From ee42efcc90873f9af8a1cecbe2c2572d809861b8 Mon Sep 17 00:00:00 2001 From: Kirill Ivanov Date: Wed, 20 Nov 2024 19:35:19 +0900 Subject: [PATCH 01/41] query table rows estimate --- stats/stats/src/charts/db_interaction/read.rs | 102 +++++++++++++++++- .../kinds/local_db/parameters/query.rs | 24 +++++ 2 files changed, 123 insertions(+), 3 deletions(-) diff --git a/stats/stats/src/charts/db_interaction/read.rs b/stats/stats/src/charts/db_interaction/read.rs index ef490c2cb..c5e7730e0 100644 --- a/stats/stats/src/charts/db_interaction/read.rs +++ b/stats/stats/src/charts/db_interaction/read.rs @@ -20,7 +20,7 @@ use itertools::Itertools; use sea_orm::{ sea_query::{self, Expr}, ColumnTrait, ConnectionTrait, DatabaseConnection, DbBackend, DbErr, EntityTrait, - FromQueryResult, QueryFilter, QueryOrder, QuerySelect, Statement, + FromQueryResult, QueryFilter, QueryOrder, QuerySelect, Statement, TryGetableMany, }; use std::{collections::HashMap, fmt::Debug, ops::Range}; use thiserror::Error; @@ -469,6 +469,45 @@ where .ok_or_else(|| DbErr::RecordNotFound("no blocks found in blockscout database".into())) } +#[derive(Debug, FromQueryResult)] +struct CountEstimate { + count: Option, +} + +/// `None` means either that +/// - db hasn't been initialized before +/// - `table_name` wasn't found +pub async fn get_estimated_table_rows( + blockscout: &DatabaseConnection, + table_name: &str, +) -> Result, DbErr> { + let statement: Statement = Statement::from_sql_and_values( + DbBackend::Postgres, + r#" + SELECT ( + CASE WHEN c.reltuples < 0 THEN + NULL + WHEN c.relpages = 0 THEN + float8 '0' + ELSE c.reltuples / c.relpages + END * + ( + pg_catalog.pg_relation_size(c.oid) / + pg_catalog.current_setting('block_size')::int + ) + )::bigint as count + FROM pg_catalog.pg_class c + WHERE c.oid = $1::regclass + "#, + vec![table_name.into()], + ); + let count = CountEstimate::find_by_statement(statement) + .one(blockscout) + .await?; + let count = count.map(|c| c.count).flatten(); + Ok(count) +} + #[derive(Debug, FromQueryResult)] struct SyncInfo { pub min_blockscout_block: Option, @@ -677,7 +716,8 @@ mod tests { data_source::kinds::local_db::parameters::DefaultQueryVec, lines::{AccountsGrowth, ActiveAccounts, TxnsGrowth, TxnsGrowthMonthly}, tests::{ - init_db::init_db, + init_db::{init_db, init_db_all}, + mock_blockscout::fill_mock_blockscout_data, point_construction::{d, month_of}, }, types::timespans::Month, @@ -686,7 +726,7 @@ mod tests { use chrono::DateTime; use entity::{chart_data, charts, sea_orm_active_enums::ChartType}; use pretty_assertions::assert_eq; - use sea_orm::{EntityTrait, Set}; + use sea_orm::{EntityName, EntityTrait, Set}; use std::str::FromStr; fn mock_chart_data(chart_id: i32, date: &str, value: i64) -> chart_data::ActiveModel { @@ -1290,4 +1330,60 @@ mod tests { }) ); } + + #[tokio::test] + #[ignore = "needs database to run"] + async fn get_estimated_table_rows_works() { + let (_db, blockscout) = init_db_all("get_estimated_table_rows_works").await; + fill_mock_blockscout_data(&blockscout, d("2023-03-01")).await; + + // need to analyze or vacuum for `reltuples` to be updated. + // source: https://www.postgresql.org/docs/9.3/planner-stats.html + let _ = blockscout + .execute(Statement::from_string(DbBackend::Postgres, "ANALYZE;")) + .await + .unwrap(); + + let blocks_estimate = get_estimated_table_rows(&blockscout, blocks::Entity.table_name()) + .await + .unwrap() + .unwrap(); + + // should be 16 rows in the table, but it's an estimate + assert!(blocks_estimate > 5); + assert!(blocks_estimate < 30); + + assert!( + get_estimated_table_rows( + &blockscout, + blockscout_db::entity::addresses::Entity.table_name() + ) + .await + .unwrap() + .unwrap() + > 0 + ); + + assert!( + get_estimated_table_rows( + &blockscout, + blockscout_db::entity::transactions::Entity.table_name() + ) + .await + .unwrap() + .unwrap() + > 0 + ); + + assert!( + get_estimated_table_rows( + &blockscout, + blockscout_db::entity::smart_contracts::Entity.table_name() + ) + .await + .unwrap() + .unwrap() + > 0 + ); + } } diff --git a/stats/stats/src/data_source/kinds/local_db/parameters/query.rs b/stats/stats/src/data_source/kinds/local_db/parameters/query.rs index ec2560b03..b48523992 100644 --- a/stats/stats/src/data_source/kinds/local_db/parameters/query.rs +++ b/stats/stats/src/data_source/kinds/local_db/parameters/query.rs @@ -89,3 +89,27 @@ impl QueryBehaviour for DefaultQueryLast { Ok(value) } } + +pub struct QueryLastWithEstimationFallback(PhantomData); + +impl QueryBehaviour for QueryLastWithEstimationFallback { + type Output = DateValue; + + async fn query_data( + cx: &UpdateContext<'_>, + _range: Option>, + ) -> Result { + let value = get_counter_data( + cx.db, + &C::name(), + Some(cx.time.date_naive()), + C::missing_date_policy(), + ) + .await? + .ok_or(UpdateError::Internal(format!( + "no data for counter '{}' was found", + C::name() + )))?; + Ok(value) + } +} From 75d883e50878f94c33569f0d305e93af291f8683 Mon Sep 17 00:00:00 2001 From: Kirill Ivanov Date: Thu, 21 Nov 2024 16:09:45 +0900 Subject: [PATCH 02/41] estimation fallback --- stats/Cargo.lock | 59 ++++++++++ stats/Cargo.toml | 2 + stats/stats/Cargo.toml | 4 + stats/stats/src/charts/db_interaction/read.rs | 2 +- .../src/data_source/kinds/local_db/mod.rs | 11 +- .../kinds/local_db/parameters/mod.rs | 4 +- .../kinds/local_db/parameters/query.rs | 102 ++++++++++++++++-- 7 files changed, 174 insertions(+), 10 deletions(-) diff --git a/stats/Cargo.lock b/stats/Cargo.lock index c08bb95ff..c40f5d773 100644 --- a/stats/Cargo.lock +++ b/stats/Cargo.lock @@ -990,6 +990,42 @@ dependencies = [ "bytes", ] +[[package]] +name = "cached" +version = "0.54.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9718806c4a2fe9e8a56fd736f97b340dd10ed1be8ed733ed50449f351dc33cae" +dependencies = [ + "ahash 0.8.11", + "async-trait", + "cached_proc_macro", + "cached_proc_macro_types", + "futures", + "hashbrown 0.14.5", + "once_cell", + "thiserror", + "tokio", + "web-time", +] + +[[package]] +name = "cached_proc_macro" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f42a145ed2d10dce2191e1dcf30cfccfea9026660e143662ba5eec4017d5daa" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn 2.0.79", +] + +[[package]] +name = "cached_proc_macro_types" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ade8366b8bd5ba243f0a58f036cc0ca8a2f069cff1a2351ef1cac6b083e16fc0" + [[package]] name = "cc" version = "1.0.98" @@ -4717,6 +4753,7 @@ dependencies = [ "blockscout-db", "blockscout-metrics-tools", "blockscout-service-launcher", + "cached", "chrono", "entity", "futures", @@ -4735,6 +4772,7 @@ dependencies = [ "tokio", "tracing", "tracing-subscriber", + "trait-variant", "tynm", "url", "wiremock", @@ -5330,6 +5368,17 @@ dependencies = [ "tracing-serde", ] +[[package]] +name = "trait-variant" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70977707304198400eb4835a78f6a9f928bf41bba420deb8fdb175cd965d77a7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.79", +] + [[package]] name = "try-lock" version = "0.2.5" @@ -5611,6 +5660,16 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "webpki-roots" version = "0.25.4" diff --git a/stats/Cargo.toml b/stats/Cargo.toml index 0a9975403..d019af282 100644 --- a/stats/Cargo.toml +++ b/stats/Cargo.toml @@ -12,7 +12,9 @@ members = [ [workspace.dependencies] blockscout-client = { git = "https://github.com/blockscout/blockscout-rs/", rev = "506b821" } blockscout-service-launcher = { version = "0.13.1" } +cached = "0.54" rstest = "0.23.0" +trait-variant = "0.1.2" wiremock = "0.6.2" # todo: update version after https://github.com/chronotope/chrono/pull/1600 diff --git a/stats/stats/Cargo.toml b/stats/stats/Cargo.toml index a99278374..c9ccdbca0 100644 --- a/stats/stats/Cargo.toml +++ b/stats/stats/Cargo.toml @@ -14,6 +14,10 @@ sea-orm = { version = "0.12", features = [ tokio = "1" thiserror = "1.0" chrono = "0.4" +cached = { workspace = true, features = [ + "async" +] } +trait-variant = { workspace = true } paste = "1.0" portrait = "0.3.0" async-trait = "0.1" diff --git a/stats/stats/src/charts/db_interaction/read.rs b/stats/stats/src/charts/db_interaction/read.rs index c5e7730e0..2bf4d5166 100644 --- a/stats/stats/src/charts/db_interaction/read.rs +++ b/stats/stats/src/charts/db_interaction/read.rs @@ -20,7 +20,7 @@ use itertools::Itertools; use sea_orm::{ sea_query::{self, Expr}, ColumnTrait, ConnectionTrait, DatabaseConnection, DbBackend, DbErr, EntityTrait, - FromQueryResult, QueryFilter, QueryOrder, QuerySelect, Statement, TryGetableMany, + FromQueryResult, QueryFilter, QueryOrder, QuerySelect, Statement, }; use std::{collections::HashMap, fmt::Debug, ops::Range}; use thiserror::Error; diff --git a/stats/stats/src/data_source/kinds/local_db/mod.rs b/stats/stats/src/data_source/kinds/local_db/mod.rs index 35798f2c0..bc74ddf6f 100644 --- a/stats/stats/src/data_source/kinds/local_db/mod.rs +++ b/stats/stats/src/data_source/kinds/local_db/mod.rs @@ -22,7 +22,7 @@ use parameters::{ }, point::PassPoint, }, - DefaultCreate, DefaultQueryLast, DefaultQueryVec, + DefaultCreate, DefaultQueryLast, DefaultQueryVec, QueryLastWithEstimationFallback, }; use sea_orm::{prelude::DateTimeUtc, DatabaseConnection, DbErr}; @@ -114,6 +114,15 @@ pub type DirectPointLocalDbChartSource = LocalDbChartSource< C, >; +pub type DirectPointLocalDbChartSourceWithEstimate = LocalDbChartSource< + Dependency, + (), + DefaultCreate, + PassPoint, + QueryLastWithEstimationFallback, + C, +>; + impl LocalDbChartSource where diff --git a/stats/stats/src/data_source/kinds/local_db/parameters/mod.rs b/stats/stats/src/data_source/kinds/local_db/parameters/mod.rs index 0b712df3e..ea2c60291 100644 --- a/stats/stats/src/data_source/kinds/local_db/parameters/mod.rs +++ b/stats/stats/src/data_source/kinds/local_db/parameters/mod.rs @@ -3,4 +3,6 @@ mod query; pub mod update; pub use create::DefaultCreate; -pub use query::{DefaultQueryLast, DefaultQueryVec}; +pub use query::{ + DefaultQueryLast, DefaultQueryVec, QueryLastWithEstimationFallback, ValueEstimation, +}; diff --git a/stats/stats/src/data_source/kinds/local_db/parameters/query.rs b/stats/stats/src/data_source/kinds/local_db/parameters/query.rs index b48523992..0d37f6c46 100644 --- a/stats/stats/src/data_source/kinds/local_db/parameters/query.rs +++ b/stats/stats/src/data_source/kinds/local_db/parameters/query.rs @@ -1,6 +1,6 @@ use std::{fmt::Debug, marker::PhantomData, ops::Range}; -use sea_orm::prelude::DateTimeUtc; +use sea_orm::{prelude::DateTimeUtc, DatabaseConnection}; use crate::{ charts::db_interaction::read::get_counter_data, @@ -90,9 +90,21 @@ impl QueryBehaviour for DefaultQueryLast { } } -pub struct QueryLastWithEstimationFallback(PhantomData); +#[trait_variant::make(Send)] +pub trait ValueEstimation { + async fn estimate(blockscout: &DatabaseConnection) -> Result, UpdateError>; +} + +pub struct QueryLastWithEstimationFallback(PhantomData<(E, C)>) +where + C: ChartProperties, + E: ValueEstimation; -impl QueryBehaviour for QueryLastWithEstimationFallback { +impl QueryBehaviour for QueryLastWithEstimationFallback +where + C: ChartProperties, + E: ValueEstimation, +{ type Output = DateValue; async fn query_data( @@ -106,10 +118,86 @@ impl QueryBehaviour for QueryLastWithEstimationFallback { C::missing_date_policy(), ) .await? - .ok_or(UpdateError::Internal(format!( - "no data for counter '{}' was found", - C::name() - )))?; + .unwrap_or(E::estimate(cx.blockscout).await?); Ok(value) } } + +#[cfg(test)] +mod tests { + use std::str::FromStr; + + use chrono::NaiveDate; + use entity::sea_orm_active_enums::ChartType; + use pretty_assertions::assert_eq; + use sea_orm::DatabaseConnection; + + use super::*; + + use crate::{ + data_source::{types::BlockscoutMigrations, UpdateContext, UpdateParameters}, + tests::init_db::init_db_all, + types::timespans::DateValue, + MissingDatePolicy, Named, UpdateError, + }; + + #[tokio::test] + #[ignore = "needs database to run"] + async fn fallback_query_works() { + let _ = tracing_subscriber::fmt::try_init(); + let (db, blockscout) = init_db_all("fallback_query_works").await; + let current_time = chrono::DateTime::from_str("2023-03-01T12:00:00Z").unwrap(); + + let parameters = UpdateParameters { + db: &db, + blockscout: &blockscout, + blockscout_applied_migrations: BlockscoutMigrations::latest(), + update_time_override: Some(current_time), + force_full: true, + }; + let cx = UpdateContext::from_params_now_or_override(parameters.clone()); + + struct TestFallback; + + fn expected_estimate() -> DateValue { + DateValue { + timespan: chrono::NaiveDate::MAX, + value: "estimate".to_string(), + } + } + + impl ValueEstimation for TestFallback { + async fn estimate( + _blockscout: &DatabaseConnection, + ) -> Result, UpdateError> { + Ok(expected_estimate()) + } + } + + pub struct InvalidProperties; + impl Named for InvalidProperties { + fn name() -> String { + "totalBlocks".into() + } + } + impl ChartProperties for InvalidProperties { + type Resolution = NaiveDate; + + fn chart_type() -> ChartType { + ChartType::Counter + } + fn missing_date_policy() -> MissingDatePolicy { + MissingDatePolicy::FillPrevious + } + } + + assert_eq!( + expected_estimate(), + QueryLastWithEstimationFallback::::query_data( + &cx, None + ) + .await + .unwrap() + ); + } +} From f0ec028753d14296d404cb9459f3add558f8c785 Mon Sep 17 00:00:00 2001 From: Kirill Ivanov Date: Thu, 21 Nov 2024 16:09:51 +0900 Subject: [PATCH 03/41] total blocks with fallback --- .../stats/src/charts/counters/total_blocks.rs | 38 +++++++++++++++++-- 1 file changed, 35 insertions(+), 3 deletions(-) diff --git a/stats/stats/src/charts/counters/total_blocks.rs b/stats/stats/src/charts/counters/total_blocks.rs index 8b14fb609..45b90ae4b 100644 --- a/stats/stats/src/charts/counters/total_blocks.rs +++ b/stats/stats/src/charts/counters/total_blocks.rs @@ -1,9 +1,10 @@ use std::ops::Range; use crate::{ + charts::db_interaction::read::get_estimated_table_rows, data_source::{ kinds::{ - local_db::DirectPointLocalDbChartSource, + local_db::{parameters::ValueEstimation, DirectPointLocalDbChartSourceWithEstimate}, remote_db::{RemoteDatabaseSource, RemoteQueryBehaviour}, }, types::UpdateContext, @@ -13,7 +14,7 @@ use crate::{ }; use blockscout_db::entity::blocks; -use chrono::{NaiveDate, NaiveDateTime}; +use chrono::{NaiveDate, NaiveDateTime, Utc}; use entity::sea_orm_active_enums::ChartType; use sea_orm::{prelude::*, sea_query::Expr, FromQueryResult, QuerySelect}; @@ -72,7 +73,38 @@ impl ChartProperties for Properties { } } -pub type TotalBlocks = DirectPointLocalDbChartSource; +pub struct CachedBlocksEstimation; + +impl ValueEstimation for CachedBlocksEstimation { + async fn estimate(blockscout: &DatabaseConnection) -> Result, UpdateError> { + #[cached::proc_macro::once(time = 60, sync_writes = true, result = true)] + async fn cached_blocks_estimation( + blockscout: &DatabaseConnection, + ) -> Result, DbErr> { + get_estimated_table_rows(blockscout, blocks::Entity.table_name()).await + } + + let now = Utc::now(); + let value = cached_blocks_estimation(blockscout) + .await + .map_err(UpdateError::BlockscoutDB)? + .map(|b| { + let b = b as f64 * 0.9; + b as i64 + }) + .unwrap_or(0); + Ok(DateValue { + timespan: now.date_naive(), + value: value.to_string(), + }) + } +} + +pub type TotalBlocks = DirectPointLocalDbChartSourceWithEstimate< + TotalBlocksRemote, + CachedBlocksEstimation, + Properties, +>; #[cfg(test)] mod tests { From 34bac891b917806eb28c970228936b66d797e122 Mon Sep 17 00:00:00 2001 From: Kirill Ivanov Date: Fri, 22 Nov 2024 14:07:01 +0900 Subject: [PATCH 04/41] add fill_missing_dates option to QueryBehaviour --- stats/stats/src/data_source/kinds/local_db/mod.rs | 4 +++- .../src/data_source/kinds/local_db/parameter_traits.rs | 1 + .../src/data_source/kinds/local_db/parameters/query.rs | 7 +++++-- .../kinds/local_db/parameters/update/batching/mod.rs | 2 ++ 4 files changed, 11 insertions(+), 3 deletions(-) diff --git a/stats/stats/src/data_source/kinds/local_db/mod.rs b/stats/stats/src/data_source/kinds/local_db/mod.rs index bc74ddf6f..4110f0e67 100644 --- a/stats/stats/src/data_source/kinds/local_db/mod.rs +++ b/stats/stats/src/data_source/kinds/local_db/mod.rs @@ -260,7 +260,9 @@ where dependency_data_fetch_timer: &mut AggregateTimer, ) -> Result { let _timer = dependency_data_fetch_timer.start_interval(); - Query::query_data(cx, range).await + // maybe add `fill_missing_dates` parameter to current function as well in the future + // to get rid of "Note" in the `DataSource`'s method documentation + Query::query_data(cx, range, false).await } } diff --git a/stats/stats/src/data_source/kinds/local_db/parameter_traits.rs b/stats/stats/src/data_source/kinds/local_db/parameter_traits.rs index 8de6b4aa2..93d8ac4ca 100644 --- a/stats/stats/src/data_source/kinds/local_db/parameter_traits.rs +++ b/stats/stats/src/data_source/kinds/local_db/parameter_traits.rs @@ -62,5 +62,6 @@ pub trait QueryBehaviour { fn query_data( cx: &UpdateContext<'_>, range: Option>, + fill_missing_dates: bool, ) -> impl Future> + Send; } diff --git a/stats/stats/src/data_source/kinds/local_db/parameters/query.rs b/stats/stats/src/data_source/kinds/local_db/parameters/query.rs index 0d37f6c46..d10a505c5 100644 --- a/stats/stats/src/data_source/kinds/local_db/parameters/query.rs +++ b/stats/stats/src/data_source/kinds/local_db/parameters/query.rs @@ -29,6 +29,7 @@ where async fn query_data( cx: &UpdateContext<'_>, range: Option>, + fill_missing_dates: bool, ) -> Result { // In DB we store data with date precision. Also, `get_line_chart_data` // works with inclusive range. Therefore, we need to convert the range and @@ -54,7 +55,7 @@ where end, None, C::missing_date_policy(), - false, + fill_missing_dates, C::approximate_trailing_points(), ) .await? @@ -74,6 +75,7 @@ impl QueryBehaviour for DefaultQueryLast { async fn query_data( cx: &UpdateContext<'_>, _range: Option>, + _fill_missing_dates: bool, ) -> Result { let value = get_counter_data( cx.db, @@ -110,6 +112,7 @@ where async fn query_data( cx: &UpdateContext<'_>, _range: Option>, + _fill_missing_dates: bool, ) -> Result { let value = get_counter_data( cx.db, @@ -194,7 +197,7 @@ mod tests { assert_eq!( expected_estimate(), QueryLastWithEstimationFallback::::query_data( - &cx, None + &cx, None, true ) .await .unwrap() diff --git a/stats/stats/src/data_source/kinds/local_db/parameters/update/batching/mod.rs b/stats/stats/src/data_source/kinds/local_db/parameters/update/batching/mod.rs index 28548d13d..72f6f4e2a 100644 --- a/stats/stats/src/data_source/kinds/local_db/parameters/update/batching/mod.rs +++ b/stats/stats/src/data_source/kinds/local_db/parameters/update/batching/mod.rs @@ -132,8 +132,10 @@ where let last_point_range_values = Query::query_data( cx, Some(previous_step_last_point_timespan.clone().into_time_range()), + false, ) .await?; + // might be replaced with `fill_missing_dates=true` let previous_step_last_point = last_point_range_values .last() .cloned() From 10c42c7062e483d230c67e3113d35f969aefd57f Mon Sep 17 00:00:00 2001 From: Kirill Ivanov Date: Fri, 22 Nov 2024 17:48:48 +0900 Subject: [PATCH 05/41] chart objects that allow specifying query behaviour for read service --- stats/stats/Cargo.toml | 1 + stats/stats/src/charts/chart.rs | 26 ++++- .../stats/src/charts/counters/total_blocks.rs | 26 +++++ stats/stats/src/charts/mod.rs | 5 +- stats/stats/src/charts/query_dispatch.rs | 108 ++++++++++++++++++ .../src/data_source/kinds/local_db/mod.rs | 5 +- stats/stats/src/lib.rs | 4 +- stats/stats/src/update_group.rs | 4 +- 8 files changed, 171 insertions(+), 8 deletions(-) create mode 100644 stats/stats/src/charts/query_dispatch.rs diff --git a/stats/stats/Cargo.toml b/stats/stats/Cargo.toml index c9ccdbca0..dce42f435 100644 --- a/stats/stats/Cargo.toml +++ b/stats/stats/Cargo.toml @@ -25,6 +25,7 @@ tracing = "0.1" tynm = "0.1.10" futures = "0.3" migration = { path = "./migration" } +stats-proto = { path = "../stats-proto" } url = "2.3" rand = "0.8" lazy_static = "1.4" diff --git a/stats/stats/src/charts/chart.rs b/stats/stats/src/charts/chart.rs index bc1f56d1b..1256a4d74 100644 --- a/stats/stats/src/charts/chart.rs +++ b/stats/stats/src/charts/chart.rs @@ -12,7 +12,10 @@ use entity::sea_orm_active_enums::{ChartResolution, ChartType}; use sea_orm::prelude::*; use thiserror::Error; -use super::db_interaction::read::ApproxUnsignedDiff; +use super::{ + db_interaction::read::ApproxUnsignedDiff, + query_dispatch::{ChartTypeSpecifics, QuerySerialized, QuerySerializedDyn}, +}; #[derive(Error, Debug)] pub enum UpdateError { @@ -232,6 +235,27 @@ macro_rules! define_and_impl_resolution_properties { }; } +/// Dynamic object representing a chart +pub struct ChartObject { + pub properties: ChartPropertiesObject, + pub type_specifics: ChartTypeSpecifics, +} + +impl ChartObject { + pub fn construct_from_chart(t: T) -> Self + where + T: ChartProperties + QuerySerialized + Send + 'static, + QuerySerializedDyn: Into, + { + Self { + properties: ChartPropertiesObject::construct_from_chart::(), + type_specifics: as Into>::into( + Box::new(t), + ), + } + } +} + /// Dynamic version of trait `ChartProperties`. /// /// Helpful when need a unified type for different charts diff --git a/stats/stats/src/charts/counters/total_blocks.rs b/stats/stats/src/charts/counters/total_blocks.rs index 45b90ae4b..830aa7176 100644 --- a/stats/stats/src/charts/counters/total_blocks.rs +++ b/stats/stats/src/charts/counters/total_blocks.rs @@ -221,4 +221,30 @@ mod tests { let data = get_raw_counters(&db).await.unwrap(); assert_eq!("13", data[&TotalBlocks::name()].value); } + + #[tokio::test] + #[ignore = "needs database to run"] + async fn total_blocks_fallback() { + let _ = tracing_subscriber::fmt::try_init(); + let (db, blockscout) = init_db_all("total_blocks_fallback").await; + let current_time = chrono::DateTime::from_str("2023-03-01T12:00:00Z").unwrap(); + let current_date = current_time.date_naive(); + + TotalBlocks::init_recursively(&db, ¤t_time) + .await + .unwrap(); + + fill_mock_blockscout_data(&blockscout, current_date).await; + + let parameters = UpdateParameters { + db: &db, + blockscout: &blockscout, + blockscout_applied_migrations: BlockscoutMigrations::latest(), + update_time_override: Some(current_time), + force_full: false, + }; + let cx: UpdateContext<'_> = UpdateContext::from_params_now_or_override(parameters.clone()); + let data = get_raw_counters(&db).await.unwrap(); + assert_eq!("13", data[&TotalBlocks::name()].value); + } } diff --git a/stats/stats/src/charts/mod.rs b/stats/stats/src/charts/mod.rs index 9b34d6c93..c7abc1d27 100644 --- a/stats/stats/src/charts/mod.rs +++ b/stats/stats/src/charts/mod.rs @@ -2,8 +2,9 @@ mod chart; pub mod counters; pub mod db_interaction; pub mod lines; +pub mod query_dispatch; pub mod types; pub use chart::{ - chart_properties_portrait, ChartKey, ChartProperties, ChartPropertiesObject, MissingDatePolicy, - Named, ResolutionKind, UpdateError, + chart_properties_portrait, ChartKey, ChartObject, ChartProperties, ChartPropertiesObject, + MissingDatePolicy, Named, ResolutionKind, UpdateError, }; diff --git a/stats/stats/src/charts/query_dispatch.rs b/stats/stats/src/charts/query_dispatch.rs new file mode 100644 index 000000000..39bb45526 --- /dev/null +++ b/stats/stats/src/charts/query_dispatch.rs @@ -0,0 +1,108 @@ +use std::{future::Future, ops::Range}; + +use chrono::{DateTime, Utc}; +use stats_proto::blockscout::stats::v1::Point; + +use crate::{ + data_source::{ + kinds::local_db::{ + parameter_traits::{CreateBehaviour, QueryBehaviour, UpdateBehaviour}, + LocalDbChartSource, + }, + DataSource, UpdateContext, + }, + exclusive_datetime_range_to_inclusive, +}; + +use super::{ + types::{ExtendedTimespanValue, Timespan}, + ChartProperties, UpdateError, +}; + +/// Data query trait with unified data format (for external use) +pub trait QuerySerialized { + /// Currently `Point` or `Vec` + type Output: Send; + + /// Retrieve chart data from local storage. + fn query_data<'a>( + &self, + cx: &UpdateContext<'a>, + range: Option>>, + fill_missing_dates: bool, + ) -> Box> + Send + 'a>; +} + +// todo: remove +/// [`QuerySerialized`] but for dynamic dispatch. +/// +/// Not `dyn QuerySerialized` because it adds unnecessary vtable lookup. +// pub type QuerySerializedMethod = Box> + Send>; + +/// [`QuerySerialized`] but for dynamic dispatch +pub type QuerySerializedDyn = Box + Send>; + +pub enum ChartTypeSpecifics { + Line { + query: QuerySerializedDyn>, + }, + Counter { + query: QuerySerializedDyn, + }, +} + +impl Into for QuerySerializedDyn { + fn into(self) -> ChartTypeSpecifics { + ChartTypeSpecifics::Counter { query: self } + } +} + +impl Into for QuerySerializedDyn> { + fn into(self) -> ChartTypeSpecifics { + ChartTypeSpecifics::Line { query: self } + } +} + +impl QuerySerialized + for LocalDbChartSource +where + MainDep: DataSource, + ResolutionDep: DataSource, + Create: CreateBehaviour, + Update: UpdateBehaviour, + Query: QueryBehaviour>>, + ChartProps: ChartProperties, +{ + type Output = Vec; + + fn query_data<'a>( + &self, + cx: &UpdateContext<'a>, + range: Option>>, + fill_missing_dates: bool, + ) -> Box> + Send + 'a> { + let cx = cx.clone(); + Box::new(async move { + let data = Query::query_data(&cx, range, fill_missing_dates).await?; + Ok(serialize_line_points(data)) + }) + } +} + +pub fn serialize_line_points( + data: Vec>, +) -> Vec { + data.into_iter() + .map(|point| { + let time_range = + exclusive_datetime_range_to_inclusive(point.timespan.into_time_range()); + let date_range = { time_range.start().date_naive()..=time_range.end().date_naive() }; + Point { + date: date_range.start().to_string(), + date_to: date_range.end().to_string(), + value: point.value, + is_approximate: point.is_approximate, + } + }) + .collect() +} diff --git a/stats/stats/src/data_source/kinds/local_db/mod.rs b/stats/stats/src/data_source/kinds/local_db/mod.rs index 4110f0e67..b13841a96 100644 --- a/stats/stats/src/data_source/kinds/local_db/mod.rs +++ b/stats/stats/src/data_source/kinds/local_db/mod.rs @@ -25,11 +25,13 @@ use parameters::{ DefaultCreate, DefaultQueryLast, DefaultQueryVec, QueryLastWithEstimationFallback, }; use sea_orm::{prelude::DateTimeUtc, DatabaseConnection, DbErr}; +use stats_proto::blockscout::stats::v1::Point; use crate::{ charts::{ chart_properties_portrait, db_interaction::read::{get_chart_metadata, get_min_block_blockscout, last_accurate_point}, + query_dispatch::QuerySerialized, ChartProperties, Named, }, data_source::{DataSource, UpdateContext}, @@ -53,7 +55,7 @@ pub mod parameters; /// /// See [module-level documentation](self) for more details. pub struct LocalDbChartSource( - PhantomData<(MainDep, ResolutionDep, Create, Update, Query, ChartProps)>, + pub PhantomData<(MainDep, ResolutionDep, Create, Update, Query, ChartProps)>, ) where MainDep: DataSource, @@ -295,7 +297,6 @@ where ChartProps: ChartProperties, { } - #[cfg(test)] mod tests { mod update_itself_is_triggered_once_per_group { diff --git a/stats/stats/src/lib.rs b/stats/stats/src/lib.rs index 6fd73cc4a..7c27eeeaf 100644 --- a/stats/stats/src/lib.rs +++ b/stats/stats/src/lib.rs @@ -18,8 +18,8 @@ pub use charts::{ db_interaction::read::{ get_line_chart_data, get_raw_counters, ApproxUnsignedDiff, ReadError, RequestedPointsLimit, }, - lines, types, ChartKey, ChartProperties, ChartPropertiesObject, MissingDatePolicy, Named, - ResolutionKind, UpdateError, + lines, types, ChartKey, ChartObject, ChartProperties, ChartPropertiesObject, MissingDatePolicy, + Named, ResolutionKind, UpdateError, }; pub use utils::exclusive_datetime_range_to_inclusive; diff --git a/stats/stats/src/update_group.rs b/stats/stats/src/update_group.rs index 6092cac53..77f046cd2 100644 --- a/stats/stats/src/update_group.rs +++ b/stats/stats/src/update_group.rs @@ -41,7 +41,7 @@ use thiserror::Error; use tokio::sync::{Mutex, MutexGuard}; use crate::{ - charts::{chart_properties_portrait::imports::ChartKey, ChartPropertiesObject}, + charts::{chart_properties_portrait::imports::ChartKey, ChartObject, ChartPropertiesObject}, data_source::UpdateParameters, UpdateError, }; @@ -263,6 +263,8 @@ macro_rules! construct_update_group { fn list_charts(&self) -> ::std::vec::Vec<$crate::ChartPropertiesObject> { std::vec![ $( + // todo: uncomment and fix type mismatch + // $crate::ChartObject::construct_from_chart::<$member>($member), $crate::ChartPropertiesObject::construct_from_chart::<$member>(), )* ] From e0b4ee2f998db82135ef386f17017bfb19b0ae79 Mon Sep 17 00:00:00 2001 From: Kirill Ivanov Date: Mon, 25 Nov 2024 16:54:14 +0900 Subject: [PATCH 06/41] construct chart objects in update groups --- .../src/charts/counters/last_new_contracts.rs | 6 +- .../counters/last_new_verified_contracts.rs | 5 +- .../src/charts/counters/total_accounts.rs | 6 +- .../stats/src/charts/counters/total_blocks.rs | 7 +- .../counters/total_native_coin_holders.rs | 5 +- .../counters/total_verified_contracts.rs | 5 +- .../stats/src/charts/lines/accounts_growth.rs | 11 +-- .../src/charts/lines/average_block_rewards.rs | 10 +-- .../src/charts/lines/average_block_size.rs | 14 ++-- .../src/charts/lines/average_gas_limit.rs | 10 +-- .../src/charts/lines/average_gas_price.rs | 10 +-- .../stats/src/charts/lines/average_txn_fee.rs | 10 +-- .../src/charts/lines/contracts_growth.rs | 10 +-- .../stats/src/charts/lines/gas_used_growth.rs | 10 +-- .../lines/native_coin_holders_growth.rs | 13 ++-- .../src/charts/lines/native_coin_supply.rs | 11 +-- stats/stats/src/charts/lines/new_accounts.rs | 6 +- .../src/charts/lines/new_block_rewards.rs | 7 +- stats/stats/src/charts/lines/new_blocks.rs | 6 +- stats/stats/src/charts/lines/new_contracts.rs | 6 +- .../charts/lines/new_native_coin_holders.rs | 6 +- .../charts/lines/new_native_coin_transfers.rs | 7 +- stats/stats/src/charts/lines/new_txns.rs | 6 +- .../charts/lines/new_verified_contracts.rs | 6 +- stats/stats/src/charts/lines/txns_fee.rs | 6 +- stats/stats/src/charts/lines/txns_growth.rs | 10 +-- .../src/charts/lines/txns_success_rate.rs | 10 +-- .../charts/lines/verified_contracts_growth.rs | 10 +-- stats/stats/src/charts/query_dispatch.rs | 72 +++++++++++++------ .../kinds/data_manipulation/last_point.rs | 3 +- .../kinds/data_manipulation/map/mod.rs | 2 + .../data_manipulation/map/strip_extension.rs | 36 ++++++++++ .../src/data_source/kinds/local_db/mod.rs | 9 ++- .../kinds/local_db/parameters/query.rs | 30 ++++---- .../parameters/update/batching/mod.rs | 26 ++++--- stats/stats/src/data_source/tests.rs | 14 ++-- stats/stats/src/update_group.rs | 22 +++--- 37 files changed, 282 insertions(+), 161 deletions(-) create mode 100644 stats/stats/src/data_source/kinds/data_manipulation/map/strip_extension.rs diff --git a/stats/stats/src/charts/counters/last_new_contracts.rs b/stats/stats/src/charts/counters/last_new_contracts.rs index 6298eb78a..efad07605 100644 --- a/stats/stats/src/charts/counters/last_new_contracts.rs +++ b/stats/stats/src/charts/counters/last_new_contracts.rs @@ -1,6 +1,7 @@ use crate::{ data_source::kinds::{ - data_manipulation::last_point::LastPoint, local_db::DirectPointLocalDbChartSource, + data_manipulation::{last_point::LastPoint, map::StripExt}, + local_db::DirectPointLocalDbChartSource, }, lines::NewContracts, ChartProperties, Named, @@ -25,7 +26,8 @@ impl ChartProperties for Properties { } } -pub type LastNewContracts = DirectPointLocalDbChartSource, Properties>; +pub type LastNewContracts = + DirectPointLocalDbChartSource>, Properties>; #[cfg(test)] mod tests { diff --git a/stats/stats/src/charts/counters/last_new_verified_contracts.rs b/stats/stats/src/charts/counters/last_new_verified_contracts.rs index bae0a5650..271af6f0f 100644 --- a/stats/stats/src/charts/counters/last_new_verified_contracts.rs +++ b/stats/stats/src/charts/counters/last_new_verified_contracts.rs @@ -1,6 +1,7 @@ use crate::{ data_source::kinds::{ - data_manipulation::last_point::LastPoint, local_db::DirectPointLocalDbChartSource, + data_manipulation::{last_point::LastPoint, map::StripExt}, + local_db::DirectPointLocalDbChartSource, }, lines::NewVerifiedContracts, ChartProperties, Named, @@ -26,7 +27,7 @@ impl ChartProperties for Properties { } pub type LastNewVerifiedContracts = - DirectPointLocalDbChartSource, Properties>; + DirectPointLocalDbChartSource>, Properties>; #[cfg(test)] mod tests { diff --git a/stats/stats/src/charts/counters/total_accounts.rs b/stats/stats/src/charts/counters/total_accounts.rs index 97c9fd8d3..2ff88c84f 100644 --- a/stats/stats/src/charts/counters/total_accounts.rs +++ b/stats/stats/src/charts/counters/total_accounts.rs @@ -1,6 +1,7 @@ use crate::{ data_source::kinds::{ - data_manipulation::last_point::LastPoint, local_db::DirectPointLocalDbChartSource, + data_manipulation::{last_point::LastPoint, map::StripExt}, + local_db::DirectPointLocalDbChartSource, }, lines::AccountsGrowth, ChartProperties, MissingDatePolicy, Named, @@ -28,7 +29,8 @@ impl ChartProperties for Properties { } } -pub type TotalAccounts = DirectPointLocalDbChartSource, Properties>; +pub type TotalAccounts = + DirectPointLocalDbChartSource>, Properties>; #[cfg(test)] mod tests { diff --git a/stats/stats/src/charts/counters/total_blocks.rs b/stats/stats/src/charts/counters/total_blocks.rs index 830aa7176..bc9707c7d 100644 --- a/stats/stats/src/charts/counters/total_blocks.rs +++ b/stats/stats/src/charts/counters/total_blocks.rs @@ -113,7 +113,7 @@ mod tests { data_source::{types::BlockscoutMigrations, DataSource, UpdateContext, UpdateParameters}, get_raw_counters, tests::{init_db::init_db_all, mock_blockscout::fill_mock_blockscout_data}, - Named, + ChartObject, Named, }; use chrono::NaiveDate; use entity::chart_data; @@ -225,6 +225,9 @@ mod tests { #[tokio::test] #[ignore = "needs database to run"] async fn total_blocks_fallback() { + // todo: finish + ChartObject::construct_from_chart::(::new_for_dynamic_dispatch()); + let _ = tracing_subscriber::fmt::try_init(); let (db, blockscout) = init_db_all("total_blocks_fallback").await; let current_time = chrono::DateTime::from_str("2023-03-01T12:00:00Z").unwrap(); @@ -245,6 +248,6 @@ mod tests { }; let cx: UpdateContext<'_> = UpdateContext::from_params_now_or_override(parameters.clone()); let data = get_raw_counters(&db).await.unwrap(); - assert_eq!("13", data[&TotalBlocks::name()].value); + // assert_eq!("13", data[&TotalBlocks::name()].value); } } diff --git a/stats/stats/src/charts/counters/total_native_coin_holders.rs b/stats/stats/src/charts/counters/total_native_coin_holders.rs index 3e40e42fc..d87acb5d2 100644 --- a/stats/stats/src/charts/counters/total_native_coin_holders.rs +++ b/stats/stats/src/charts/counters/total_native_coin_holders.rs @@ -1,6 +1,7 @@ use crate::{ data_source::kinds::{ - data_manipulation::last_point::LastPoint, local_db::DirectPointLocalDbChartSource, + data_manipulation::{last_point::LastPoint, map::StripExt}, + local_db::DirectPointLocalDbChartSource, }, lines::NativeCoinHoldersGrowth, ChartProperties, MissingDatePolicy, Named, @@ -29,7 +30,7 @@ impl ChartProperties for Properties { } pub type TotalNativeCoinHolders = - DirectPointLocalDbChartSource, Properties>; + DirectPointLocalDbChartSource>, Properties>; #[cfg(test)] mod tests { diff --git a/stats/stats/src/charts/counters/total_verified_contracts.rs b/stats/stats/src/charts/counters/total_verified_contracts.rs index 38eb0880d..6acf9712c 100644 --- a/stats/stats/src/charts/counters/total_verified_contracts.rs +++ b/stats/stats/src/charts/counters/total_verified_contracts.rs @@ -1,6 +1,7 @@ use crate::{ data_source::kinds::{ - data_manipulation::last_point::LastPoint, local_db::DirectPointLocalDbChartSource, + data_manipulation::{last_point::LastPoint, map::StripExt}, + local_db::DirectPointLocalDbChartSource, }, lines::VerifiedContractsGrowth, ChartProperties, MissingDatePolicy, Named, @@ -29,7 +30,7 @@ impl ChartProperties for Properties { } pub type TotalVerifiedContracts = - DirectPointLocalDbChartSource, Properties>; + DirectPointLocalDbChartSource>, Properties>; #[cfg(test)] mod tests { diff --git a/stats/stats/src/charts/lines/accounts_growth.rs b/stats/stats/src/charts/lines/accounts_growth.rs index 6c5410402..246c3b5ce 100644 --- a/stats/stats/src/charts/lines/accounts_growth.rs +++ b/stats/stats/src/charts/lines/accounts_growth.rs @@ -3,7 +3,7 @@ use super::new_accounts::NewAccountsInt; use crate::{ data_source::kinds::{ - data_manipulation::resolutions::last_value::LastValueLowerResolution, + data_manipulation::{map::StripExt, resolutions::last_value::LastValueLowerResolution}, local_db::{ parameters::update::batching::parameters::{Batch30Weeks, Batch30Years, Batch36Months}, DailyCumulativeLocalDbChartSource, DirectVecLocalDbChartSource, @@ -46,18 +46,21 @@ define_and_impl_resolution_properties!( ); pub type AccountsGrowth = DailyCumulativeLocalDbChartSource; +type AccountsGrowthS = StripExt; + pub type AccountsGrowthWeekly = DirectVecLocalDbChartSource< - LastValueLowerResolution, + LastValueLowerResolution, Batch30Weeks, WeeklyProperties, >; pub type AccountsGrowthMonthly = DirectVecLocalDbChartSource< - LastValueLowerResolution, + LastValueLowerResolution, Batch36Months, MonthlyProperties, >; +type AccountsGrowthMonthlyS = StripExt; pub type AccountsGrowthYearly = DirectVecLocalDbChartSource< - LastValueLowerResolution, + LastValueLowerResolution, Batch30Years, YearlyProperties, >; diff --git a/stats/stats/src/charts/lines/average_block_rewards.rs b/stats/stats/src/charts/lines/average_block_rewards.rs index 7377eb225..439cebfd4 100644 --- a/stats/stats/src/charts/lines/average_block_rewards.rs +++ b/stats/stats/src/charts/lines/average_block_rewards.rs @@ -4,7 +4,7 @@ use crate::{ data_source::{ kinds::{ data_manipulation::{ - map::{MapParseTo, MapToString}, + map::{MapParseTo, MapToString, StripExt}, resolutions::average::AverageLowerResolution, }, local_db::{ @@ -87,24 +87,26 @@ define_and_impl_resolution_properties!( pub type AverageBlockRewards = DirectVecLocalDbChartSource; +type AverageBlockRewardsS = StripExt; pub type AverageBlockRewardsWeekly = DirectVecLocalDbChartSource< MapToString< - AverageLowerResolution, NewBlockRewardsInt, Week>, + AverageLowerResolution, NewBlockRewardsInt, Week>, >, Batch30Weeks, WeeklyProperties, >; pub type AverageBlockRewardsMonthly = DirectVecLocalDbChartSource< MapToString< - AverageLowerResolution, NewBlockRewardsInt, Month>, + AverageLowerResolution, NewBlockRewardsInt, Month>, >, Batch36Months, MonthlyProperties, >; +type AverageBlockRewardsMonthlyS = StripExt; pub type AverageBlockRewardsYearly = DirectVecLocalDbChartSource< MapToString< AverageLowerResolution< - MapParseTo, + MapParseTo, NewBlockRewardsMonthlyInt, Year, >, diff --git a/stats/stats/src/charts/lines/average_block_size.rs b/stats/stats/src/charts/lines/average_block_size.rs index ef04fb15c..848703271 100644 --- a/stats/stats/src/charts/lines/average_block_size.rs +++ b/stats/stats/src/charts/lines/average_block_size.rs @@ -4,7 +4,7 @@ use crate::{ data_source::{ kinds::{ data_manipulation::{ - map::{MapParseTo, MapToString}, + map::{MapParseTo, MapToString, StripExt}, resolutions::average::AverageLowerResolution, }, local_db::{ @@ -82,20 +82,26 @@ define_and_impl_resolution_properties!( pub type AverageBlockSize = DirectVecLocalDbChartSource; +type AverageBlockSizeS = StripExt; pub type AverageBlockSizeWeekly = DirectVecLocalDbChartSource< - MapToString, NewBlocksInt, Week>>, + MapToString, NewBlocksInt, Week>>, Batch30Weeks, WeeklyProperties, >; pub type AverageBlockSizeMonthly = DirectVecLocalDbChartSource< - MapToString, NewBlocksInt, Month>>, + MapToString, NewBlocksInt, Month>>, Batch36Months, MonthlyProperties, >; +type AverageBlockSizeMonthlyS = StripExt; pub type AverageBlockSizeYearly = DirectVecLocalDbChartSource< MapToString< - AverageLowerResolution, NewBlocksMonthlyInt, Year>, + AverageLowerResolution< + MapParseTo, + NewBlocksMonthlyInt, + Year, + >, >, Batch30Years, YearlyProperties, diff --git a/stats/stats/src/charts/lines/average_gas_limit.rs b/stats/stats/src/charts/lines/average_gas_limit.rs index 53111726b..d9d762675 100644 --- a/stats/stats/src/charts/lines/average_gas_limit.rs +++ b/stats/stats/src/charts/lines/average_gas_limit.rs @@ -4,7 +4,7 @@ use crate::{ data_source::{ kinds::{ data_manipulation::{ - map::{MapParseTo, MapToString}, + map::{MapParseTo, MapToString, StripExt}, resolutions::average::AverageLowerResolution, }, local_db::{ @@ -82,19 +82,21 @@ define_and_impl_resolution_properties!( pub type AverageGasLimit = DirectVecLocalDbChartSource; +type AverageGasLimitS = StripExt; pub type AverageGasLimitWeekly = DirectVecLocalDbChartSource< - MapToString, NewBlocksInt, Week>>, + MapToString, NewBlocksInt, Week>>, Batch30Weeks, WeeklyProperties, >; pub type AverageGasLimitMonthly = DirectVecLocalDbChartSource< - MapToString, NewBlocksInt, Month>>, + MapToString, NewBlocksInt, Month>>, Batch36Months, MonthlyProperties, >; +type AverageGasLimitMonthlyS = StripExt; pub type AverageGasLimitYearly = DirectVecLocalDbChartSource< MapToString< - AverageLowerResolution, NewBlocksMonthlyInt, Year>, + AverageLowerResolution, NewBlocksMonthlyInt, Year>, >, Batch30Years, YearlyProperties, diff --git a/stats/stats/src/charts/lines/average_gas_price.rs b/stats/stats/src/charts/lines/average_gas_price.rs index f1a8b31ee..0ab1afec5 100644 --- a/stats/stats/src/charts/lines/average_gas_price.rs +++ b/stats/stats/src/charts/lines/average_gas_price.rs @@ -4,7 +4,7 @@ use crate::{ data_source::{ kinds::{ data_manipulation::{ - map::{MapParseTo, MapToString}, + map::{MapParseTo, MapToString, StripExt}, resolutions::average::AverageLowerResolution, }, local_db::{ @@ -137,19 +137,21 @@ define_and_impl_resolution_properties!( pub type AverageGasPrice = DirectVecLocalDbChartSource; +type AverageGasPriceS = StripExt; pub type AverageGasPriceWeekly = DirectVecLocalDbChartSource< - MapToString, NewTxnsInt, Week>>, + MapToString, NewTxnsInt, Week>>, Batch30Weeks, WeeklyProperties, >; pub type AverageGasPriceMonthly = DirectVecLocalDbChartSource< - MapToString, NewTxnsInt, Month>>, + MapToString, NewTxnsInt, Month>>, Batch36Months, MonthlyProperties, >; +type AverageGasPriceMonthlyS = StripExt; pub type AverageGasPriceYearly = DirectVecLocalDbChartSource< MapToString< - AverageLowerResolution, NewTxnsMonthlyInt, Year>, + AverageLowerResolution, NewTxnsMonthlyInt, Year>, >, Batch30Years, YearlyProperties, diff --git a/stats/stats/src/charts/lines/average_txn_fee.rs b/stats/stats/src/charts/lines/average_txn_fee.rs index 7d0d7b058..4d3cb09f1 100644 --- a/stats/stats/src/charts/lines/average_txn_fee.rs +++ b/stats/stats/src/charts/lines/average_txn_fee.rs @@ -6,7 +6,7 @@ use crate::{ data_source::{ kinds::{ data_manipulation::{ - map::{MapParseTo, MapToString}, + map::{MapParseTo, MapToString, StripExt}, resolutions::average::AverageLowerResolution, }, local_db::{ @@ -140,19 +140,21 @@ define_and_impl_resolution_properties!( pub type AverageTxnFee = DirectVecLocalDbChartSource; +type AverageTxnFeeS = StripExt; pub type AverageTxnFeeWeekly = DirectVecLocalDbChartSource< - MapToString, NewTxnsInt, Week>>, + MapToString, NewTxnsInt, Week>>, Batch30Weeks, WeeklyProperties, >; pub type AverageTxnFeeMonthly = DirectVecLocalDbChartSource< - MapToString, NewTxnsInt, Month>>, + MapToString, NewTxnsInt, Month>>, Batch36Months, MonthlyProperties, >; +type AverageTxnFeeMonthlyS = StripExt; pub type AverageTxnFeeYearly = DirectVecLocalDbChartSource< MapToString< - AverageLowerResolution, NewTxnsMonthlyInt, Year>, + AverageLowerResolution, NewTxnsMonthlyInt, Year>, >, Batch30Years, YearlyProperties, diff --git a/stats/stats/src/charts/lines/contracts_growth.rs b/stats/stats/src/charts/lines/contracts_growth.rs index 47c32b118..9a3703f32 100644 --- a/stats/stats/src/charts/lines/contracts_growth.rs +++ b/stats/stats/src/charts/lines/contracts_growth.rs @@ -1,6 +1,6 @@ use crate::{ data_source::kinds::{ - data_manipulation::resolutions::last_value::LastValueLowerResolution, + data_manipulation::{map::StripExt, resolutions::last_value::LastValueLowerResolution}, local_db::{ parameters::update::batching::parameters::{Batch30Weeks, Batch30Years, Batch36Months}, DailyCumulativeLocalDbChartSource, DirectVecLocalDbChartSource, @@ -44,18 +44,20 @@ define_and_impl_resolution_properties!( ); pub type ContractsGrowth = DailyCumulativeLocalDbChartSource; +type ContractsGrowthS = StripExt; pub type ContractsGrowthWeekly = DirectVecLocalDbChartSource< - LastValueLowerResolution, + LastValueLowerResolution, Batch30Weeks, WeeklyProperties, >; pub type ContractsGrowthMonthly = DirectVecLocalDbChartSource< - LastValueLowerResolution, + LastValueLowerResolution, Batch36Months, MonthlyProperties, >; +type ContractsGrowthMonthlyS = StripExt; pub type ContractsGrowthYearly = DirectVecLocalDbChartSource< - LastValueLowerResolution, + LastValueLowerResolution, Batch30Years, YearlyProperties, >; diff --git a/stats/stats/src/charts/lines/gas_used_growth.rs b/stats/stats/src/charts/lines/gas_used_growth.rs index 8680f2a36..3ff92ba6b 100644 --- a/stats/stats/src/charts/lines/gas_used_growth.rs +++ b/stats/stats/src/charts/lines/gas_used_growth.rs @@ -4,7 +4,7 @@ use crate::{ data_source::{ kinds::{ data_manipulation::{ - map::{Map, MapFunction}, + map::{Map, MapFunction, StripExt}, resolutions::last_value::LastValueLowerResolution, }, local_db::{ @@ -102,18 +102,20 @@ define_and_impl_resolution_properties!( ); pub type GasUsedGrowth = DailyCumulativeLocalDbChartSource; +type GasUsedGrowthS = StripExt; pub type GasUsedGrowthWeekly = DirectVecLocalDbChartSource< - LastValueLowerResolution, + LastValueLowerResolution, Batch30Weeks, WeeklyProperties, >; pub type GasUsedGrowthMonthly = DirectVecLocalDbChartSource< - LastValueLowerResolution, + LastValueLowerResolution, Batch36Months, MonthlyProperties, >; +type GasUsedGrowthMonthlyS = StripExt; pub type GasUsedGrowthYearly = DirectVecLocalDbChartSource< - LastValueLowerResolution, + LastValueLowerResolution, Batch30Years, YearlyProperties, >; diff --git a/stats/stats/src/charts/lines/native_coin_holders_growth.rs b/stats/stats/src/charts/lines/native_coin_holders_growth.rs index d7fcd4667..d1ac2e802 100644 --- a/stats/stats/src/charts/lines/native_coin_holders_growth.rs +++ b/stats/stats/src/charts/lines/native_coin_holders_growth.rs @@ -5,7 +5,8 @@ use crate::{ data_source::{ kinds::{ data_manipulation::{ - map::MapParseTo, resolutions::last_value::LastValueLowerResolution, + map::{MapParseTo, StripExt}, + resolutions::last_value::LastValueLowerResolution, }, local_db::{ parameter_traits::{CreateBehaviour, UpdateBehaviour}, @@ -384,19 +385,21 @@ define_and_impl_resolution_properties!( pub type NativeCoinHoldersGrowth = LocalDbChartSource<(), (), Create, Update, DefaultQueryVec, Properties>; -pub type NativeCoinHoldersGrowthInt = MapParseTo; +pub type NativeCoinHoldersGrowthInt = MapParseTo, i64>; +type NativeCoinHoldersGrowthS = StripExt; pub type NativeCoinHoldersGrowthWeekly = DirectVecLocalDbChartSource< - LastValueLowerResolution, + LastValueLowerResolution, Batch30Weeks, WeeklyProperties, >; pub type NativeCoinHoldersGrowthMonthly = DirectVecLocalDbChartSource< - LastValueLowerResolution, + LastValueLowerResolution, Batch36Months, MonthlyProperties, >; +type NativeCoinHoldersGrowthMonthlyS = StripExt; pub type NativeCoinHoldersGrowthYearly = DirectVecLocalDbChartSource< - LastValueLowerResolution, + LastValueLowerResolution, Batch30Years, YearlyProperties, >; diff --git a/stats/stats/src/charts/lines/native_coin_supply.rs b/stats/stats/src/charts/lines/native_coin_supply.rs index 8b86f6a6e..6fccfb654 100644 --- a/stats/stats/src/charts/lines/native_coin_supply.rs +++ b/stats/stats/src/charts/lines/native_coin_supply.rs @@ -4,7 +4,8 @@ use crate::{ data_source::{ kinds::{ data_manipulation::{ - map::MapToString, resolutions::last_value::LastValueLowerResolution, + map::{MapToString, StripExt}, + resolutions::last_value::LastValueLowerResolution, }, local_db::{ parameters::update::batching::parameters::{ @@ -120,18 +121,20 @@ define_and_impl_resolution_properties!( pub type NativeCoinSupply = DirectVecLocalDbChartSource; +type NativeCoinSupplyS = StripExt; pub type NativeCoinSupplyWeekly = DirectVecLocalDbChartSource< - LastValueLowerResolution, + LastValueLowerResolution, Batch30Weeks, WeeklyProperties, >; pub type NativeCoinSupplyMonthly = DirectVecLocalDbChartSource< - LastValueLowerResolution, + LastValueLowerResolution, Batch36Months, MonthlyProperties, >; +type NativeCoinSupplyMonthlyS = StripExt; pub type NativeCoinSupplyYearly = DirectVecLocalDbChartSource< - LastValueLowerResolution, + LastValueLowerResolution, Batch30Years, YearlyProperties, >; diff --git a/stats/stats/src/charts/lines/new_accounts.rs b/stats/stats/src/charts/lines/new_accounts.rs index 0644a955c..553d0e410 100644 --- a/stats/stats/src/charts/lines/new_accounts.rs +++ b/stats/stats/src/charts/lines/new_accounts.rs @@ -5,7 +5,7 @@ use crate::{ data_source::{ kinds::{ data_manipulation::{ - map::{MapParseTo, MapToString}, + map::{MapParseTo, MapToString, StripExt}, resolutions::sum::SumLowerResolution, }, local_db::{ @@ -150,7 +150,7 @@ define_and_impl_resolution_properties!( ); pub type NewAccounts = DirectVecLocalDbChartSource; -pub type NewAccountsInt = MapParseTo; +pub type NewAccountsInt = MapParseTo, i64>; pub type NewAccountsWeekly = DirectVecLocalDbChartSource< MapToString>, Batch30Weeks, @@ -161,7 +161,7 @@ pub type NewAccountsMonthly = DirectVecLocalDbChartSource< Batch36Months, MonthlyProperties, >; -pub type NewAccountsMonthlyInt = MapParseTo; +pub type NewAccountsMonthlyInt = MapParseTo, i64>; pub type NewAccountsYearly = DirectVecLocalDbChartSource< MapToString>, Batch30Years, diff --git a/stats/stats/src/charts/lines/new_block_rewards.rs b/stats/stats/src/charts/lines/new_block_rewards.rs index f3adf5755..cc0e469c5 100644 --- a/stats/stats/src/charts/lines/new_block_rewards.rs +++ b/stats/stats/src/charts/lines/new_block_rewards.rs @@ -8,7 +8,10 @@ use std::ops::Range; use crate::{ data_source::{ kinds::{ - data_manipulation::{map::MapParseTo, resolutions::sum::SumLowerResolution}, + data_manipulation::{ + map::{MapParseTo, StripExt}, + resolutions::sum::SumLowerResolution, + }, local_db::{ parameters::update::batching::parameters::Batch30Days, DirectVecLocalDbChartSource, }, @@ -80,7 +83,7 @@ define_and_impl_resolution_properties!( pub type NewBlockRewards = DirectVecLocalDbChartSource; -pub type NewBlockRewardsInt = MapParseTo; +pub type NewBlockRewardsInt = MapParseTo, i64>; pub type NewBlockRewardsMonthlyInt = SumLowerResolution; #[cfg(test)] diff --git a/stats/stats/src/charts/lines/new_blocks.rs b/stats/stats/src/charts/lines/new_blocks.rs index b1d5b4c8c..1e35fbe08 100644 --- a/stats/stats/src/charts/lines/new_blocks.rs +++ b/stats/stats/src/charts/lines/new_blocks.rs @@ -4,7 +4,7 @@ use crate::{ data_source::{ kinds::{ data_manipulation::{ - map::{MapParseTo, MapToString}, + map::{MapParseTo, MapToString, StripExt}, resolutions::sum::SumLowerResolution, }, local_db::{ @@ -79,7 +79,7 @@ define_and_impl_resolution_properties!( ); pub type NewBlocks = DirectVecLocalDbChartSource; -pub type NewBlocksInt = MapParseTo; +pub type NewBlocksInt = MapParseTo, i64>; pub type NewBlocksWeekly = DirectVecLocalDbChartSource< MapToString>, Batch30Weeks, @@ -90,7 +90,7 @@ pub type NewBlocksMonthly = DirectVecLocalDbChartSource< Batch36Months, MonthlyProperties, >; -pub type NewBlocksMonthlyInt = MapParseTo; +pub type NewBlocksMonthlyInt = MapParseTo, i64>; pub type NewBlocksYearly = DirectVecLocalDbChartSource< MapToString>, Batch30Years, diff --git a/stats/stats/src/charts/lines/new_contracts.rs b/stats/stats/src/charts/lines/new_contracts.rs index 3f91a7727..00226bb50 100644 --- a/stats/stats/src/charts/lines/new_contracts.rs +++ b/stats/stats/src/charts/lines/new_contracts.rs @@ -4,7 +4,7 @@ use crate::{ data_source::{ kinds::{ data_manipulation::{ - map::{MapParseTo, MapToString}, + map::{MapParseTo, MapToString, StripExt}, resolutions::sum::SumLowerResolution, }, local_db::{ @@ -145,7 +145,7 @@ define_and_impl_resolution_properties!( ); pub type NewContracts = DirectVecLocalDbChartSource; -pub type NewContractsInt = MapParseTo; +pub type NewContractsInt = MapParseTo, i64>; pub type NewContractsWeekly = DirectVecLocalDbChartSource< MapToString>, Batch30Weeks, @@ -156,7 +156,7 @@ pub type NewContractsMonthly = DirectVecLocalDbChartSource< Batch36Months, MonthlyProperties, >; -pub type NewContractsMonthlyInt = MapParseTo; +pub type NewContractsMonthlyInt = MapParseTo, i64>; pub type NewContractsYearly = DirectVecLocalDbChartSource< MapToString>, Batch30Years, diff --git a/stats/stats/src/charts/lines/new_native_coin_holders.rs b/stats/stats/src/charts/lines/new_native_coin_holders.rs index 797942bdd..d764e071a 100644 --- a/stats/stats/src/charts/lines/new_native_coin_holders.rs +++ b/stats/stats/src/charts/lines/new_native_coin_holders.rs @@ -3,7 +3,7 @@ use crate::{ data_source::kinds::{ data_manipulation::{ delta::Delta, - map::{MapParseTo, MapToString}, + map::{MapParseTo, MapToString, StripExt}, resolutions::sum::SumLowerResolution, }, local_db::{ @@ -52,7 +52,7 @@ pub type NewNativeCoinHolders = DirectVecLocalDbChartSource< Batch30Days, Properties, >; -pub type NewNativeCoinHoldersInt = MapParseTo; +pub type NewNativeCoinHoldersInt = MapParseTo, i64>; pub type NewNativeCoinHoldersWeekly = DirectVecLocalDbChartSource< MapToString>, Batch30Weeks, @@ -63,7 +63,7 @@ pub type NewNativeCoinHoldersMonthly = DirectVecLocalDbChartSource< Batch36Months, MonthlyProperties, >; -pub type NewNativeCoinHoldersMonthlyInt = MapParseTo; +pub type NewNativeCoinHoldersMonthlyInt = MapParseTo, i64>; pub type NewNativeCoinHoldersYearly = DirectVecLocalDbChartSource< MapToString>, Batch30Years, diff --git a/stats/stats/src/charts/lines/new_native_coin_transfers.rs b/stats/stats/src/charts/lines/new_native_coin_transfers.rs index 7612143ed..7f89afff1 100644 --- a/stats/stats/src/charts/lines/new_native_coin_transfers.rs +++ b/stats/stats/src/charts/lines/new_native_coin_transfers.rs @@ -4,7 +4,7 @@ use crate::{ data_source::{ kinds::{ data_manipulation::{ - map::{MapParseTo, MapToString}, + map::{MapParseTo, MapToString, StripExt}, resolutions::sum::SumLowerResolution, }, local_db::{ @@ -107,7 +107,7 @@ define_and_impl_resolution_properties!( pub type NewNativeCoinTransfers = DirectVecLocalDbChartSource; -pub type NewNativeCoinTransfersInt = MapParseTo; +pub type NewNativeCoinTransfersInt = MapParseTo, i64>; pub type NewNativeCoinTransfersWeekly = DirectVecLocalDbChartSource< MapToString>, Batch30Weeks, @@ -118,7 +118,8 @@ pub type NewNativeCoinTransfersMonthly = DirectVecLocalDbChartSource< Batch36Months, MonthlyProperties, >; -pub type NewNativeCoinTransfersMonthlyInt = MapParseTo; +pub type NewNativeCoinTransfersMonthlyInt = + MapParseTo, i64>; pub type NewNativeCoinTransfersYearly = DirectVecLocalDbChartSource< MapToString>, Batch30Years, diff --git a/stats/stats/src/charts/lines/new_txns.rs b/stats/stats/src/charts/lines/new_txns.rs index 76a6029bb..6dec3c470 100644 --- a/stats/stats/src/charts/lines/new_txns.rs +++ b/stats/stats/src/charts/lines/new_txns.rs @@ -4,7 +4,7 @@ use crate::{ data_source::{ kinds::{ data_manipulation::{ - map::{MapParseTo, MapToString}, + map::{MapParseTo, MapToString, StripExt}, resolutions::sum::SumLowerResolution, }, local_db::{ @@ -102,7 +102,7 @@ define_and_impl_resolution_properties!( ); pub type NewTxns = DirectVecLocalDbChartSource; -pub type NewTxnsInt = MapParseTo; +pub type NewTxnsInt = MapParseTo, i64>; pub type NewTxnsWeekly = DirectVecLocalDbChartSource< MapToString>, Batch30Weeks, @@ -113,7 +113,7 @@ pub type NewTxnsMonthly = DirectVecLocalDbChartSource< Batch36Months, MonthlyProperties, >; -pub type NewTxnsMonthlyInt = MapParseTo; +pub type NewTxnsMonthlyInt = MapParseTo, i64>; pub type NewTxnsYearly = DirectVecLocalDbChartSource< MapToString>, Batch30Years, diff --git a/stats/stats/src/charts/lines/new_verified_contracts.rs b/stats/stats/src/charts/lines/new_verified_contracts.rs index e4a0c1e6b..211b8644f 100644 --- a/stats/stats/src/charts/lines/new_verified_contracts.rs +++ b/stats/stats/src/charts/lines/new_verified_contracts.rs @@ -4,7 +4,7 @@ use crate::{ data_source::{ kinds::{ data_manipulation::{ - map::{MapParseTo, MapToString}, + map::{MapParseTo, MapToString, StripExt}, resolutions::sum::SumLowerResolution, }, local_db::{ @@ -78,7 +78,7 @@ define_and_impl_resolution_properties!( pub type NewVerifiedContracts = DirectVecLocalDbChartSource; -pub type NewVerifiedContractsInt = MapParseTo; +pub type NewVerifiedContractsInt = MapParseTo, i64>; pub type NewVerifiedContractsWeekly = DirectVecLocalDbChartSource< MapToString>, Batch30Weeks, @@ -89,7 +89,7 @@ pub type NewVerifiedContractsMonthly = DirectVecLocalDbChartSource< Batch36Months, MonthlyProperties, >; -pub type NewVerifiedContractsMonthlyInt = MapParseTo; +pub type NewVerifiedContractsMonthlyInt = MapParseTo, i64>; pub type NewVerifiedContractsYearly = DirectVecLocalDbChartSource< MapToString>, Batch30Years, diff --git a/stats/stats/src/charts/lines/txns_fee.rs b/stats/stats/src/charts/lines/txns_fee.rs index 5a93a2362..aabe549bc 100644 --- a/stats/stats/src/charts/lines/txns_fee.rs +++ b/stats/stats/src/charts/lines/txns_fee.rs @@ -6,7 +6,7 @@ use crate::{ data_source::{ kinds::{ data_manipulation::{ - map::{MapParseTo, MapToString}, + map::{MapParseTo, MapToString, StripExt}, resolutions::sum::SumLowerResolution, }, local_db::{ @@ -136,7 +136,7 @@ define_and_impl_resolution_properties!( ); pub type TxnsFee = DirectVecLocalDbChartSource; -pub type TxnsFeeFloat = MapParseTo; +pub type TxnsFeeFloat = MapParseTo, f64>; pub type TxnsFeeWeekly = DirectVecLocalDbChartSource< MapToString>, Batch30Weeks, @@ -147,7 +147,7 @@ pub type TxnsFeeMonthly = DirectVecLocalDbChartSource< Batch36Months, MonthlyProperties, >; -pub type TxnsFeeMonthlyFloat = MapParseTo; +pub type TxnsFeeMonthlyFloat = MapParseTo, f64>; pub type TxnsFeeYearly = DirectVecLocalDbChartSource< MapToString>, Batch30Years, diff --git a/stats/stats/src/charts/lines/txns_growth.rs b/stats/stats/src/charts/lines/txns_growth.rs index bbffcab5d..344776fa8 100644 --- a/stats/stats/src/charts/lines/txns_growth.rs +++ b/stats/stats/src/charts/lines/txns_growth.rs @@ -1,7 +1,7 @@ use crate::{ charts::chart::ChartProperties, data_source::kinds::{ - data_manipulation::resolutions::last_value::LastValueLowerResolution, + data_manipulation::{map::StripExt, resolutions::last_value::LastValueLowerResolution}, local_db::{ parameters::update::batching::parameters::{Batch30Weeks, Batch30Years, Batch36Months}, DailyCumulativeLocalDbChartSource, DirectVecLocalDbChartSource, @@ -45,18 +45,20 @@ define_and_impl_resolution_properties!( ); pub type TxnsGrowth = DailyCumulativeLocalDbChartSource; +type TxnsGrowthS = StripExt; pub type TxnsGrowthWeekly = DirectVecLocalDbChartSource< - LastValueLowerResolution, + LastValueLowerResolution, Batch30Weeks, WeeklyProperties, >; pub type TxnsGrowthMonthly = DirectVecLocalDbChartSource< - LastValueLowerResolution, + LastValueLowerResolution, Batch36Months, MonthlyProperties, >; +type TxnsGrowthMonthlyS = StripExt; pub type TxnsGrowthYearly = DirectVecLocalDbChartSource< - LastValueLowerResolution, + LastValueLowerResolution, Batch30Years, YearlyProperties, >; diff --git a/stats/stats/src/charts/lines/txns_success_rate.rs b/stats/stats/src/charts/lines/txns_success_rate.rs index 3bb543605..71ab391d6 100644 --- a/stats/stats/src/charts/lines/txns_success_rate.rs +++ b/stats/stats/src/charts/lines/txns_success_rate.rs @@ -4,7 +4,7 @@ use crate::{ data_source::{ kinds::{ data_manipulation::{ - map::{MapParseTo, MapToString}, + map::{MapParseTo, MapToString, StripExt}, resolutions::average::AverageLowerResolution, }, local_db::{ @@ -113,19 +113,21 @@ define_and_impl_resolution_properties!( pub type TxnsSuccessRate = DirectVecLocalDbChartSource; +type TxnsSuccessRateS = StripExt; pub type TxnsSuccessRateWeekly = DirectVecLocalDbChartSource< - MapToString, NewTxnsInt, Week>>, + MapToString, NewTxnsInt, Week>>, Batch30Weeks, WeeklyProperties, >; pub type TxnsSuccessRateMonthly = DirectVecLocalDbChartSource< - MapToString, NewTxnsInt, Month>>, + MapToString, NewTxnsInt, Month>>, Batch36Months, MonthlyProperties, >; +type TxnsSuccessRateMonthlyS = StripExt; pub type TxnsSuccessRateYearly = DirectVecLocalDbChartSource< MapToString< - AverageLowerResolution, NewTxnsMonthlyInt, Year>, + AverageLowerResolution, NewTxnsMonthlyInt, Year>, >, Batch30Years, YearlyProperties, diff --git a/stats/stats/src/charts/lines/verified_contracts_growth.rs b/stats/stats/src/charts/lines/verified_contracts_growth.rs index 44b2f2dc1..6a2100771 100644 --- a/stats/stats/src/charts/lines/verified_contracts_growth.rs +++ b/stats/stats/src/charts/lines/verified_contracts_growth.rs @@ -1,7 +1,7 @@ use crate::{ charts::chart::ChartProperties, data_source::kinds::{ - data_manipulation::resolutions::last_value::LastValueLowerResolution, + data_manipulation::{map::StripExt, resolutions::last_value::LastValueLowerResolution}, local_db::{ parameters::update::batching::parameters::{Batch30Weeks, Batch30Years, Batch36Months}, DailyCumulativeLocalDbChartSource, DirectVecLocalDbChartSource, @@ -46,18 +46,20 @@ define_and_impl_resolution_properties!( pub type VerifiedContractsGrowth = DailyCumulativeLocalDbChartSource; +type VerifiedContractsGrowthS = StripExt; pub type VerifiedContractsGrowthWeekly = DirectVecLocalDbChartSource< - LastValueLowerResolution, + LastValueLowerResolution, Batch30Weeks, WeeklyProperties, >; pub type VerifiedContractsGrowthMonthly = DirectVecLocalDbChartSource< - LastValueLowerResolution, + LastValueLowerResolution, Batch36Months, MonthlyProperties, >; +type VerifiedContractsGrowthMonthlyS = StripExt; pub type VerifiedContractsGrowthYearly = DirectVecLocalDbChartSource< - LastValueLowerResolution, + LastValueLowerResolution, Batch30Years, YearlyProperties, >; diff --git a/stats/stats/src/charts/query_dispatch.rs b/stats/stats/src/charts/query_dispatch.rs index 39bb45526..809564d0d 100644 --- a/stats/stats/src/charts/query_dispatch.rs +++ b/stats/stats/src/charts/query_dispatch.rs @@ -1,6 +1,6 @@ use std::{future::Future, ops::Range}; -use chrono::{DateTime, Utc}; +use chrono::{DateTime, NaiveDate, Utc}; use stats_proto::blockscout::stats::v1::Point; use crate::{ @@ -15,7 +15,7 @@ use crate::{ }; use super::{ - types::{ExtendedTimespanValue, Timespan}, + types::{ExtendedTimespanValue, Timespan, TimespanValue}, ChartProperties, UpdateError, }; @@ -43,15 +43,15 @@ pub trait QuerySerialized { pub type QuerySerializedDyn = Box + Send>; pub enum ChartTypeSpecifics { + Counter { + query: QuerySerializedDyn>, + }, Line { query: QuerySerializedDyn>, }, - Counter { - query: QuerySerializedDyn, - }, } -impl Into for QuerySerializedDyn { +impl Into for QuerySerializedDyn> { fn into(self) -> ChartTypeSpecifics { ChartTypeSpecifics::Counter { query: self } } @@ -63,17 +63,42 @@ impl Into for QuerySerializedDyn> { } } -impl QuerySerialized +pub trait SerializableQueryOutput { + type Serialized; + fn serialize(self) -> Self::Serialized; +} + +impl SerializableQueryOutput + for Vec> +{ + type Serialized = Vec; + + fn serialize(self) -> Self::Serialized { + serialize_line_points(self) + } +} + +impl SerializableQueryOutput for TimespanValue { + type Serialized = TimespanValue; + + fn serialize(self) -> Self::Serialized { + self + } +} + +impl QuerySerialized for LocalDbChartSource where MainDep: DataSource, ResolutionDep: DataSource, Create: CreateBehaviour, Update: UpdateBehaviour, - Query: QueryBehaviour>>, + Query: QueryBehaviour, + QueryOutput: SerializableQueryOutput, + QueryOutput::Serialized: Send, ChartProps: ChartProperties, { - type Output = Vec; + type Output = QueryOutput::Serialized; fn query_data<'a>( &self, @@ -84,25 +109,26 @@ where let cx = cx.clone(); Box::new(async move { let data = Query::query_data(&cx, range, fill_missing_dates).await?; - Ok(serialize_line_points(data)) + Ok(data.serialize()) }) } } +fn serialize_point( + point: ExtendedTimespanValue, +) -> Point { + let time_range = exclusive_datetime_range_to_inclusive(point.timespan.into_time_range()); + let date_range = { time_range.start().date_naive()..=time_range.end().date_naive() }; + Point { + date: date_range.start().to_string(), + date_to: date_range.end().to_string(), + value: point.value, + is_approximate: point.is_approximate, + } +} + pub fn serialize_line_points( data: Vec>, ) -> Vec { - data.into_iter() - .map(|point| { - let time_range = - exclusive_datetime_range_to_inclusive(point.timespan.into_time_range()); - let date_range = { time_range.start().date_naive()..=time_range.end().date_naive() }; - Point { - date: date_range.start().to_string(), - date_to: date_range.end().to_string(), - value: point.value, - is_approximate: point.is_approximate, - } - }) - .collect() + data.into_iter().map(serialize_point).collect() } diff --git a/stats/stats/src/data_source/kinds/data_manipulation/last_point.rs b/stats/stats/src/data_source/kinds/data_manipulation/last_point.rs index ff98bfcf1..036651aad 100644 --- a/stats/stats/src/data_source/kinds/data_manipulation/last_point.rs +++ b/stats/stats/src/data_source/kinds/data_manipulation/last_point.rs @@ -9,7 +9,6 @@ use chrono::{DateTime, Utc}; use sea_orm::{prelude::DateTimeUtc, DatabaseConnection, DbErr}; use crate::{ - charts::ChartProperties, data_source::{source::DataSource, UpdateContext}, types::{Timespan, TimespanValue, ZeroTimespanValue}, utils::day_start, @@ -24,7 +23,7 @@ impl DataSource for LastPoint where Resolution: Timespan + Ord + Send, Value: Send, - DS: DataSource>> + ChartProperties, + DS: DataSource>>, TimespanValue: ZeroTimespanValue, { type MainDependencies = DS; diff --git a/stats/stats/src/data_source/kinds/data_manipulation/map/mod.rs b/stats/stats/src/data_source/kinds/data_manipulation/map/mod.rs index e48fffc0f..648a86a6f 100644 --- a/stats/stats/src/data_source/kinds/data_manipulation/map/mod.rs +++ b/stats/stats/src/data_source/kinds/data_manipulation/map/mod.rs @@ -13,9 +13,11 @@ use crate::{ }; mod parse; +mod strip_extension; mod to_string; pub use parse::MapParseTo; +pub use strip_extension::StripExt; pub use to_string::MapToString; /// Apply `F` to each value queried from data source `D` diff --git a/stats/stats/src/data_source/kinds/data_manipulation/map/strip_extension.rs b/stats/stats/src/data_source/kinds/data_manipulation/map/strip_extension.rs new file mode 100644 index 000000000..80de37e0f --- /dev/null +++ b/stats/stats/src/data_source/kinds/data_manipulation/map/strip_extension.rs @@ -0,0 +1,36 @@ +use crate::{ + types::{ExtendedTimespanValue, TimespanValue}, + UpdateError, +}; + +use super::{Map, MapFunction}; + +/// Remove `Extended` part from `ExtendedTimespanValue`. +/// Used because it's easier to impl and use other data source modifiers +/// this way. +pub struct StripExtensionFunction; + +impl MapFunction>> for StripExtensionFunction +where + R: Send, + V: Send, +{ + type Output = Vec>; + fn function(inner_data: Vec>) -> Result { + Ok(inner_data.into_iter().map(|p| p.into()).collect()) + } +} + +impl MapFunction> for StripExtensionFunction +where + R: Send, + V: Send, +{ + type Output = TimespanValue; + fn function(inner_data: ExtendedTimespanValue) -> Result { + Ok(inner_data.into()) + } +} + +/// Remove `Extended` part from `ExtendedTimespanValue`(-s) +pub type StripExt = Map; diff --git a/stats/stats/src/data_source/kinds/local_db/mod.rs b/stats/stats/src/data_source/kinds/local_db/mod.rs index b13841a96..26f84f0e5 100644 --- a/stats/stats/src/data_source/kinds/local_db/mod.rs +++ b/stats/stats/src/data_source/kinds/local_db/mod.rs @@ -25,13 +25,11 @@ use parameters::{ DefaultCreate, DefaultQueryLast, DefaultQueryVec, QueryLastWithEstimationFallback, }; use sea_orm::{prelude::DateTimeUtc, DatabaseConnection, DbErr}; -use stats_proto::blockscout::stats::v1::Point; use crate::{ charts::{ chart_properties_portrait, db_interaction::read::{get_chart_metadata, get_min_block_blockscout, last_accurate_point}, - query_dispatch::QuerySerialized, ChartProperties, Named, }, data_source::{DataSource, UpdateContext}, @@ -136,6 +134,12 @@ where ChartProps: ChartProperties, ChartProps::Resolution: Ord + Clone + Debug, { + /// `new` function that is created solely for the purposes of + /// dynamic dispatch (see where it's used). + pub fn new_for_dynamic_dispatch() -> Self { + Self(PhantomData) + } + /// Performs common checks and prepares values useful for further /// update. Then proceeds to update according to parameters. async fn update_itself_inner( @@ -297,6 +301,7 @@ where ChartProps: ChartProperties, { } + #[cfg(test)] mod tests { mod update_itself_is_triggered_once_per_group { diff --git a/stats/stats/src/data_source/kinds/local_db/parameters/query.rs b/stats/stats/src/data_source/kinds/local_db/parameters/query.rs index d10a505c5..707cadcf3 100644 --- a/stats/stats/src/data_source/kinds/local_db/parameters/query.rs +++ b/stats/stats/src/data_source/kinds/local_db/parameters/query.rs @@ -6,7 +6,7 @@ use crate::{ charts::db_interaction::read::get_counter_data, data_source::{kinds::local_db::parameter_traits::QueryBehaviour, UpdateContext}, get_line_chart_data, - types::{timespans::DateValue, Timespan, TimespanValue}, + types::{timespans::DateValue, ExtendedTimespanValue, Timespan}, utils::exclusive_datetime_range_to_inclusive, ChartProperties, UpdateError, }; @@ -19,7 +19,7 @@ where C: ChartProperties, C::Resolution: Timespan + Ord + Debug + Clone + Send, { - type Output = Vec>; + type Output = Vec>; /// Retrieve chart data from local storage. /// @@ -47,21 +47,17 @@ where // same for weeks or other resolutions. let start = start.map(|s| C::Resolution::from_date(s.date_naive())); let end = end.map(|e| C::Resolution::from_date(e.date_naive())); - let values: Vec> = - get_line_chart_data::( - cx.db, - &C::name(), - start, - end, - None, - C::missing_date_policy(), - fill_missing_dates, - C::approximate_trailing_points(), - ) - .await? - .into_iter() - .map(TimespanValue::from) - .collect(); + let values = get_line_chart_data::( + cx.db, + &C::name(), + start, + end, + None, + C::missing_date_policy(), + fill_missing_dates, + C::approximate_trailing_points(), + ) + .await?; Ok(values) } } diff --git a/stats/stats/src/data_source/kinds/local_db/parameters/update/batching/mod.rs b/stats/stats/src/data_source/kinds/local_db/parameters/update/batching/mod.rs index 72f6f4e2a..fc1195f4a 100644 --- a/stats/stats/src/data_source/kinds/local_db/parameters/update/batching/mod.rs +++ b/stats/stats/src/data_source/kinds/local_db/parameters/update/batching/mod.rs @@ -17,7 +17,7 @@ use crate::{ types::Get, UpdateContext, }, - types::{Timespan, TimespanDuration, TimespanValue}, + types::{ExtendedTimespanValue, Timespan, TimespanDuration, TimespanValue}, ChartProperties, UpdateError, }; @@ -39,7 +39,7 @@ where ResolutionDep: DataSource, BatchStep: BatchStepBehaviour, BatchSizeUpperBound: Get>, - Query: QueryBehaviour>>, + Query: QueryBehaviour>>, ChartProps: ChartProperties; impl @@ -50,7 +50,7 @@ where ResolutionDep: DataSource, BatchStep: BatchStepBehaviour, BatchSizeUpperBound: Get>, - Query: QueryBehaviour>>, + Query: QueryBehaviour>>, ChartProps: ChartProperties, ChartProps::Resolution: Timespan + Ord + Clone + Debug + Send, { @@ -126,7 +126,7 @@ async fn get_previous_step_last_point( ) -> Result, UpdateError> where Resolution: Timespan + Clone, - Query: QueryBehaviour>>, + Query: QueryBehaviour>>, { let previous_step_last_point_timespan = this_step_start.saturating_previous_timespan(); let last_point_range_values = Query::query_data( @@ -139,6 +139,7 @@ where let previous_step_last_point = last_point_range_values .last() .cloned() + .map(|p| p.into()) // `None` means // - if `MissingDatePolicy` is `FillZero` = the value is 0 // - if `MissingDatePolicy` is `FillPrevious` = no previous value was found at this moment in time = value at the point is 0 @@ -380,12 +381,15 @@ mod tests { use crate::{ data_source::{ - kinds::local_db::{ - parameters::{ - update::batching::{parameters::mock::RecordingPassStep, BatchUpdate}, - DefaultCreate, DefaultQueryVec, + kinds::{ + data_manipulation::map::StripExt, + local_db::{ + parameters::{ + update::batching::{parameters::mock::RecordingPassStep, BatchUpdate}, + DefaultCreate, DefaultQueryVec, + }, + LocalDbChartSource, }, - LocalDbChartSource, }, types::Get, }, @@ -434,7 +438,7 @@ mod tests { } type ThisRecordingBatchUpdate = BatchUpdate< - AccountsGrowth, + StripExt, (), ThisRecordingStep, Batch1Day, @@ -443,7 +447,7 @@ mod tests { >; type RecordingChart = LocalDbChartSource< - AccountsGrowth, + StripExt, (), DefaultCreate, ThisRecordingBatchUpdate, diff --git a/stats/stats/src/data_source/tests.rs b/stats/stats/src/data_source/tests.rs index d224c9764..839333976 100644 --- a/stats/stats/src/data_source/tests.rs +++ b/stats/stats/src/data_source/tests.rs @@ -7,7 +7,10 @@ use tokio::sync::Mutex; use super::{ kinds::{ - data_manipulation::{map::MapParseTo, resolutions::last_value::LastValueLowerResolution}, + data_manipulation::{ + map::{MapParseTo, StripExt}, + resolutions::last_value::LastValueLowerResolution, + }, local_db::{ parameters::{ update::batching::{ @@ -147,7 +150,7 @@ impl ChartProperties for NewContractsChartProperties { pub type NewContracts = DirectVecLocalDbChartSource; -pub type NewContractsInt = MapParseTo; +pub type NewContractsInt = MapParseTo, i64>; pub struct ContractsGrowthProperties; @@ -180,18 +183,19 @@ define_and_impl_resolution_properties!( pub type ContractsGrowth = DailyCumulativeLocalDbChartSource; +type ContractsGrowthS = StripExt; pub type ContractsGrowthWeekly = DirectVecLocalDbChartSource< - LastValueLowerResolution, + LastValueLowerResolution, Batch30Weeks, ContractsGrowthWeeklyProperties, >; pub type ContractsGrowthMonthly = DirectVecLocalDbChartSource< - LastValueLowerResolution, + LastValueLowerResolution, Batch36Months, ContractsGrowthMonthlyProperties, >; pub type ContractsGrowthYearly = DirectVecLocalDbChartSource< - LastValueLowerResolution, + LastValueLowerResolution, Batch30Years, ContractsGrowthYearlyProperties, >; diff --git a/stats/stats/src/update_group.rs b/stats/stats/src/update_group.rs index 77f046cd2..7787c2f4a 100644 --- a/stats/stats/src/update_group.rs +++ b/stats/stats/src/update_group.rs @@ -41,7 +41,7 @@ use thiserror::Error; use tokio::sync::{Mutex, MutexGuard}; use crate::{ - charts::{chart_properties_portrait::imports::ChartKey, ChartObject, ChartPropertiesObject}, + charts::{chart_properties_portrait::imports::ChartKey, ChartObject}, data_source::UpdateParameters, UpdateError, }; @@ -68,7 +68,7 @@ pub trait UpdateGroup: core::fmt::Debug { /// Group name (usually equal to type name for simplicity) fn name(&self) -> String; /// List chart properties - members of the group. - fn list_charts(&self) -> Vec; + fn list_charts(&self) -> Vec; /// List mutex ids of group members + their dependencies. /// Dependencies participate in updates, thus access to them needs to be /// synchronized as well. @@ -122,7 +122,7 @@ pub trait UpdateGroup: core::fmt::Debug { /// # kinds::{ /// # local_db::{DirectVecLocalDbChartSource, parameters::update::batching::parameters::Batch30Days}, /// # remote_db::{PullAllWithAndSort, RemoteDatabaseSource, StatementFromRange}, -/// # data_manipulation::map::MapToString, +/// # data_manipulation::map::{MapToString, StripExt}, /// # }, /// # types::{UpdateContext, UpdateParameters}, /// # }; @@ -147,7 +147,7 @@ pub trait UpdateGroup: core::fmt::Debug { /// # } /// # } /// # -/// # type DummyChart = DirectVecLocalDbChartSource; +/// # type DummyChart = DirectVecLocalDbChartSource, Batch30Days, DummyChartProperties>; /// /// construct_update_group!(ExampleUpdateGroup { /// charts: [DummyChart], @@ -260,12 +260,10 @@ macro_rules! construct_update_group { stringify!($group_name).into() } - fn list_charts(&self) -> ::std::vec::Vec<$crate::ChartPropertiesObject> { + fn list_charts(&self) -> ::std::vec::Vec<$crate::ChartObject> { std::vec![ $( - // todo: uncomment and fix type mismatch - // $crate::ChartObject::construct_from_chart::<$member>($member), - $crate::ChartPropertiesObject::construct_from_chart::<$member>(), + $crate::ChartObject::construct_from_chart::<$member>(<$member>::new_for_dynamic_dispatch()), )* ] } @@ -393,7 +391,7 @@ impl SyncUpdateGroup { } /// See [`UpdateGroup::list_charts``] - pub fn list_charts(&self) -> Vec { + pub fn list_charts(&self) -> Vec { self.inner.list_charts() } @@ -463,7 +461,11 @@ impl SyncUpdateGroup { &self, enabled_charts: &HashSet, ) -> (Vec>, HashSet) { - let members: HashSet = self.list_charts().into_iter().map(|c| c.key).collect(); + let members: HashSet = self + .list_charts() + .into_iter() + .map(|c| c.properties.key) + .collect(); // in-place intersection let enabled_members: HashSet = members .into_iter() From a891a32a7c63aacd84d1d9d0f8f726e0ec9cf8e2 Mon Sep 17 00:00:00 2001 From: Kirill Ivanov Date: Mon, 25 Nov 2024 19:21:58 +0900 Subject: [PATCH 07/41] drop-in place chart objects in stats-server + make it compileable --- stats/Cargo.lock | 1 + stats/stats-server/src/read_service.rs | 16 ++- stats/stats-server/src/runtime_setup.rs | 121 ++++++++++++++++------- stats/stats-server/src/serializers.rs | 22 ----- stats/stats/src/charts/chart.rs | 3 +- stats/stats/src/charts/query_dispatch.rs | 19 ++-- stats/stats/src/lib.rs | 4 +- 7 files changed, 106 insertions(+), 80 deletions(-) diff --git a/stats/Cargo.lock b/stats/Cargo.lock index c40f5d773..ceb09f7f3 100644 --- a/stats/Cargo.lock +++ b/stats/Cargo.lock @@ -4768,6 +4768,7 @@ dependencies = [ "rust_decimal", "rust_decimal_macros", "sea-orm", + "stats-proto", "thiserror", "tokio", "tracing", diff --git a/stats/stats-server/src/read_service.rs b/stats/stats-server/src/read_service.rs index 7f8dc8fbe..836522de6 100644 --- a/stats/stats-server/src/read_service.rs +++ b/stats/stats-server/src/read_service.rs @@ -3,7 +3,6 @@ use std::{clone::Clone, cmp::Ord, collections::BTreeMap, fmt::Debug, str::FromSt use crate::{ config::{layout::sorted_items_according_to_layout, types}, runtime_setup::{EnabledChartEntry, RuntimeSetup}, - serializers::serialize_line_points, settings::LimitsSettings, }; @@ -13,6 +12,7 @@ use proto_v1::stats_service_server::StatsService; use sea_orm::{DatabaseConnection, DbErr}; use stats::{ entity::sea_orm_active_enums::ChartType, + query_dispatch::serialize_line_points, types::{ timespans::{Month, Week, Year}, Timespan, @@ -69,11 +69,11 @@ fn map_read_error(err: ReadError) -> Status { /// Returns `None` if info were not found for some chart. fn add_chart_info_to_layout( layout: Vec, - chart_info: BTreeMap, + chart_info: &BTreeMap, ) -> Vec { layout .into_iter() - .map(|cat| cat.intersect_info(&chart_info)) + .map(|cat| cat.intersect_info(chart_info)) .collect() } @@ -194,7 +194,7 @@ impl StatsService for ReadService { .iter() .filter(|(_, chart)| { chart - .enabled_resolutions + .resolutions .iter() .all(|(_, static_info)| static_info.chart_type == ChartType::Counter) }) @@ -202,8 +202,7 @@ impl StatsService for ReadService { data.remove(name).and_then(|point| { // resolutions other than day are currently not supported // for counters - let Some(static_info) = counter.enabled_resolutions.get(&ResolutionKind::Day) - else { + let Some(static_info) = counter.resolutions.get(&ResolutionKind::Day) else { tracing::warn!( "No 'day' resolution enabled for counter {}, skipping its value", name @@ -244,7 +243,7 @@ impl StatsService for ReadService { Status::not_found(format!("chart with name '{}' was not found", chart_name)) })?; let resolution_info = chart_entry - .enabled_resolutions + .resolutions .get(&resolution) .filter(|static_info| static_info.chart_type == ChartType::Line) .ok_or_else(|| { @@ -285,8 +284,7 @@ impl StatsService for ReadService { _request: Request, ) -> Result, Status> { let layout = self.charts.lines_layout.clone(); - let info = self.charts.charts_info.clone(); - let sections = add_chart_info_to_layout(layout, info); + let sections = add_chart_info_to_layout(layout, &self.charts.charts_info); Ok(Response::new(proto_v1::LineCharts { sections })) } diff --git a/stats/stats-server/src/runtime_setup.rs b/stats/stats-server/src/runtime_setup.rs index e257aea6a..04f6fa3ff 100644 --- a/stats/stats-server/src/runtime_setup.rs +++ b/stats/stats-server/src/runtime_setup.rs @@ -22,8 +22,9 @@ use cron::Schedule; use itertools::Itertools; use stats::{ entity::sea_orm_active_enums::ChartType, + query_dispatch::ChartTypeSpecifics, update_group::{ArcUpdateGroup, SyncUpdateGroup}, - ChartKey, ChartPropertiesObject, ResolutionKind, + ChartKey, ChartObject, ResolutionKind, }; use std::{ collections::{btree_map::Entry, BTreeMap, HashMap, HashSet}, @@ -31,11 +32,11 @@ use std::{ }; use tokio::sync::Mutex; -#[derive(Clone, Debug)] +#[derive(Debug)] pub struct EnabledChartEntry { pub settings: EnabledChartSettings, /// Static information presented as dynamic object - pub enabled_resolutions: HashMap, + pub resolutions: HashMap, } impl EnabledChartEntry { @@ -50,7 +51,7 @@ impl EnabledChartEntry { description: settings.description, units: settings.units, resolutions: self - .enabled_resolutions + .resolutions .keys() .map(|r| String::from(*r)) .collect_vec(), @@ -58,21 +59,28 @@ impl EnabledChartEntry { } } -#[derive(Clone, Debug)] +#[derive(Debug)] pub struct EnabledResolutionEntry { pub name: String, + // todo: remove pub chart_type: ChartType, pub missing_date_policy: stats::MissingDatePolicy, pub approximate_trailing_points: u64, + pub handle: ChartTypeSpecifics, } -impl From for EnabledResolutionEntry { - fn from(value: ChartPropertiesObject) -> Self { +impl From for EnabledResolutionEntry { + fn from(value: ChartObject) -> Self { + let ChartObject { + properties: props, + type_specifics, + } = value; Self { - name: value.name, - chart_type: value.chart_type, - missing_date_policy: value.missing_date_policy, - approximate_trailing_points: value.approximate_trailing_points, + name: props.name, + chart_type: props.chart_type, + missing_date_policy: props.missing_date_policy, + approximate_trailing_points: props.approximate_trailing_points, + handle: type_specifics, } } } @@ -111,6 +119,11 @@ fn combine_disjoint_maps( Ok(a) } +struct AllChartsInfo { + counters: BTreeMap, + line_charts: BTreeMap, +} + impl RuntimeSetup { pub fn new( charts: config::charts::Config, @@ -139,10 +152,10 @@ impl RuntimeSetup { /// /// `Err(Vec)` - some unknown charts+resolutions are present in settings fn charts_info_from_settings( + available_resolutions: &mut BTreeMap, charts_settings: BTreeMap, settings_chart_type: ChartType, ) -> Result, Vec> { - let available_resolutions = Self::all_members(); let mut unknown_charts = vec![]; let mut charts_info = BTreeMap::new(); @@ -152,10 +165,14 @@ impl RuntimeSetup { let mut enabled_resolutions_properties = HashMap::new(); for (resolution, resolution_setting) in settings.resolutions.into_list() { let key = ChartKey::new(name.clone(), resolution); - let resolution_properties = available_resolutions - .get(&key) - .filter(|props| props.chart_type == settings_chart_type) - .cloned(); + let resolution_properties = match available_resolutions.entry(key.clone()) { + Entry::Occupied(o) + if o.get().properties.chart_type == settings_chart_type => + { + Some(o.remove()) + } + _ => None, + }; match (resolution_setting, resolution_properties) { // enabled (Some(true), Some(enabled_props)) | (None, Some(enabled_props)) => { @@ -172,7 +189,7 @@ impl RuntimeSetup { name, EnabledChartEntry { settings: enabled_chart_settings, - enabled_resolutions: enabled_resolutions_properties, + resolutions: enabled_resolutions_properties, }, ); } @@ -185,15 +202,26 @@ impl RuntimeSetup { } } - fn build_charts_info( - charts_config: config::charts::Config, - ) -> anyhow::Result> { - let counters_info = - Self::charts_info_from_settings(charts_config.counters, ChartType::Counter); - let lines_info = Self::charts_info_from_settings(charts_config.lines, ChartType::Line); - - let (counters_info, lines_info) = match (counters_info, lines_info) { - (Ok(c), Ok(l)) => (c, l), + fn all_charts_info_from_settings( + counters_settings: BTreeMap, + line_charts_settings: BTreeMap, + ) -> Result> { + let mut available_resolutions = Self::all_members(); + let counters_info = Self::charts_info_from_settings( + &mut available_resolutions, + counters_settings, + ChartType::Counter, + ); + let lines_info = Self::charts_info_from_settings( + &mut available_resolutions, + line_charts_settings, + ChartType::Line, + ); + match (counters_info, lines_info) { + (Ok(c), Ok(l)) => Ok(AllChartsInfo { + counters: c, + line_charts: l, + }), (counters_result, lines_result) => { let mut unknown_charts = vec![]; if let Err(c) = counters_result { @@ -202,13 +230,25 @@ impl RuntimeSetup { if let Err(l) = lines_result { unknown_charts.extend(l); } - return Err(anyhow::anyhow!( - "non-existent charts+resolutions are present in settings: {unknown_charts:?}", - )); + Err(unknown_charts) } - }; + } + } + + fn build_charts_info( + charts_config: config::charts::Config, + ) -> anyhow::Result> { + let AllChartsInfo { + counters, + line_charts, + } = Self::all_charts_info_from_settings(charts_config.counters, charts_config.lines) + .map_err(|unknown_charts| { + anyhow::anyhow!( + "non-existent charts+resolutions are present in settings: {unknown_charts:?}", + ) + })?; - combine_disjoint_maps(counters_info, lines_info) + combine_disjoint_maps(counters, line_charts) .map_err(|duplicate_name| anyhow::anyhow!("duplicate chart name: {duplicate_name:?}",)) } @@ -333,7 +373,7 @@ impl RuntimeSetup { let members: HashSet = group .list_charts() .into_iter() - .map(|c| c.key.as_string()) + .map(|c| c.properties.key.as_string()) .collect(); let missing_members: HashSet = sync_dependencies.difference(&members).cloned().collect(); @@ -393,10 +433,10 @@ impl RuntimeSetup { .into_iter() .filter(|m| { charts_info - .get(m.key.name()) - .is_some_and(|a| a.enabled_resolutions.contains_key(m.key.resolution())) + .get(m.properties.key.name()) + .is_some_and(|a| a.resolutions.contains_key(m.properties.key.resolution())) }) - .map(|m| m.key) + .map(|m| m.properties.key) .collect(); let sync_group = SyncUpdateGroup::new(&dep_mutexes, group)?; result.insert( @@ -412,14 +452,14 @@ impl RuntimeSetup { } /// List all charts+resolutions that are members of at least 1 group. - fn all_members() -> BTreeMap { + fn all_members() -> BTreeMap { let members_with_duplicates = Self::all_update_groups() .into_iter() .flat_map(|g| g.list_charts()) .collect_vec(); let mut members = BTreeMap::new(); for member in members_with_duplicates { - match members.entry(member.key.clone()) { + match members.entry(member.properties.key.clone()) { Entry::Vacant(v) => { v.insert(member); } @@ -430,7 +470,12 @@ impl RuntimeSetup { // i.e. this check does not guarantee that same mutex will not be // used for 2 different charts (although it shouldn't lead to logical // errors) - assert_eq!(o.get(), &member, "duplicate member key '{}'", o.get().key); + assert_eq!( + o.get().properties, + member.properties, + "duplicate member key '{}'", + o.get().properties.key + ); } } } diff --git a/stats/stats-server/src/serializers.rs b/stats/stats-server/src/serializers.rs index f3379bf69..8b1378917 100644 --- a/stats/stats-server/src/serializers.rs +++ b/stats/stats-server/src/serializers.rs @@ -1,23 +1 @@ -use stats::{ - exclusive_datetime_range_to_inclusive, - types::{ExtendedTimespanValue, Timespan}, -}; -use stats_proto::blockscout::stats::v1::Point; -pub fn serialize_line_points( - data: Vec>, -) -> Vec { - data.into_iter() - .map(|point| { - let time_range = - exclusive_datetime_range_to_inclusive(point.timespan.into_time_range()); - let date_range = { time_range.start().date_naive()..=time_range.end().date_naive() }; - Point { - date: date_range.start().to_string(), - date_to: date_range.end().to_string(), - value: point.value, - is_approximate: point.is_approximate, - } - }) - .collect() -} diff --git a/stats/stats/src/charts/chart.rs b/stats/stats/src/charts/chart.rs index 1256a4d74..0a21481de 100644 --- a/stats/stats/src/charts/chart.rs +++ b/stats/stats/src/charts/chart.rs @@ -236,6 +236,7 @@ macro_rules! define_and_impl_resolution_properties { } /// Dynamic object representing a chart +#[derive(Debug)] pub struct ChartObject { pub properties: ChartPropertiesObject, pub type_specifics: ChartTypeSpecifics, @@ -250,7 +251,7 @@ impl ChartObject { Self { properties: ChartPropertiesObject::construct_from_chart::(), type_specifics: as Into>::into( - Box::new(t), + std::sync::Arc::new(Box::new(t)), ), } } diff --git a/stats/stats/src/charts/query_dispatch.rs b/stats/stats/src/charts/query_dispatch.rs index 809564d0d..a13085e44 100644 --- a/stats/stats/src/charts/query_dispatch.rs +++ b/stats/stats/src/charts/query_dispatch.rs @@ -1,4 +1,4 @@ -use std::{future::Future, ops::Range}; +use std::{fmt::Debug, future::Future, ops::Range, sync::Arc}; use chrono::{DateTime, NaiveDate, Utc}; use stats_proto::blockscout::stats::v1::Point; @@ -33,14 +33,8 @@ pub trait QuerySerialized { ) -> Box> + Send + 'a>; } -// todo: remove -/// [`QuerySerialized`] but for dynamic dispatch. -/// -/// Not `dyn QuerySerialized` because it adds unnecessary vtable lookup. -// pub type QuerySerializedMethod = Box> + Send>; - /// [`QuerySerialized`] but for dynamic dispatch -pub type QuerySerializedDyn = Box + Send>; +pub type QuerySerializedDyn = Arc + Send + Sync>>; pub enum ChartTypeSpecifics { Counter { @@ -51,6 +45,15 @@ pub enum ChartTypeSpecifics { }, } +impl Debug for ChartTypeSpecifics { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Counter { query: _ } => write!(f, "Counter"), + Self::Line { query: _ } => write!(f, "Line"), + } + } +} + impl Into for QuerySerializedDyn> { fn into(self) -> ChartTypeSpecifics { ChartTypeSpecifics::Counter { query: self } diff --git a/stats/stats/src/lib.rs b/stats/stats/src/lib.rs index 7c27eeeaf..b4d49407c 100644 --- a/stats/stats/src/lib.rs +++ b/stats/stats/src/lib.rs @@ -18,8 +18,8 @@ pub use charts::{ db_interaction::read::{ get_line_chart_data, get_raw_counters, ApproxUnsignedDiff, ReadError, RequestedPointsLimit, }, - lines, types, ChartKey, ChartObject, ChartProperties, ChartPropertiesObject, MissingDatePolicy, - Named, ResolutionKind, UpdateError, + lines, query_dispatch, types, ChartKey, ChartObject, ChartProperties, ChartPropertiesObject, + MissingDatePolicy, Named, ResolutionKind, UpdateError, }; pub use utils::exclusive_datetime_range_to_inclusive; From 62f1dbfe995b3c072f7c9294ef42100bf1e16614 Mon Sep 17 00:00:00 2001 From: Kirill Ivanov Date: Mon, 25 Nov 2024 19:33:14 +0900 Subject: [PATCH 08/41] deduplicate chart type data --- stats/stats-server/src/read_service.rs | 9 ++++----- stats/stats-server/src/runtime_setup.rs | 9 +++------ stats/stats/src/charts/chart.rs | 16 +++++++++++----- stats/stats/src/charts/query_dispatch.rs | 10 ++++++++++ 4 files changed, 28 insertions(+), 16 deletions(-) diff --git a/stats/stats-server/src/read_service.rs b/stats/stats-server/src/read_service.rs index 836522de6..6e513cc10 100644 --- a/stats/stats-server/src/read_service.rs +++ b/stats/stats-server/src/read_service.rs @@ -193,10 +193,9 @@ impl StatsService for ReadService { .charts_info .iter() .filter(|(_, chart)| { - chart - .resolutions - .iter() - .all(|(_, static_info)| static_info.chart_type == ChartType::Counter) + chart.resolutions.iter().all(|(_, static_info)| { + static_info.type_specifics.as_chart_type() == ChartType::Counter + }) }) .filter_map(|(name, counter)| { data.remove(name).and_then(|point| { @@ -245,7 +244,7 @@ impl StatsService for ReadService { let resolution_info = chart_entry .resolutions .get(&resolution) - .filter(|static_info| static_info.chart_type == ChartType::Line) + .filter(|static_info| static_info.type_specifics.as_chart_type() == ChartType::Line) .ok_or_else(|| { Status::not_found(format!( "resolution '{}' for chart '{}' was not found", diff --git a/stats/stats-server/src/runtime_setup.rs b/stats/stats-server/src/runtime_setup.rs index 04f6fa3ff..d9d07b979 100644 --- a/stats/stats-server/src/runtime_setup.rs +++ b/stats/stats-server/src/runtime_setup.rs @@ -62,11 +62,9 @@ impl EnabledChartEntry { #[derive(Debug)] pub struct EnabledResolutionEntry { pub name: String, - // todo: remove - pub chart_type: ChartType, pub missing_date_policy: stats::MissingDatePolicy, pub approximate_trailing_points: u64, - pub handle: ChartTypeSpecifics, + pub type_specifics: ChartTypeSpecifics, } impl From for EnabledResolutionEntry { @@ -77,10 +75,9 @@ impl From for EnabledResolutionEntry { } = value; Self { name: props.name, - chart_type: props.chart_type, missing_date_policy: props.missing_date_policy, approximate_trailing_points: props.approximate_trailing_points, - handle: type_specifics, + type_specifics, } } } @@ -167,7 +164,7 @@ impl RuntimeSetup { let key = ChartKey::new(name.clone(), resolution); let resolution_properties = match available_resolutions.entry(key.clone()) { Entry::Occupied(o) - if o.get().properties.chart_type == settings_chart_type => + if o.get().type_specifics.as_chart_type() == settings_chart_type => { Some(o.remove()) } diff --git a/stats/stats/src/charts/chart.rs b/stats/stats/src/charts/chart.rs index 0a21481de..4bca1bf7a 100644 --- a/stats/stats/src/charts/chart.rs +++ b/stats/stats/src/charts/chart.rs @@ -9,6 +9,7 @@ use std::fmt::Display; use crate::{types::Timespan, ReadError}; use chrono::{DateTime, Utc}; use entity::sea_orm_active_enums::{ChartResolution, ChartType}; +use pretty_assertions::assert_eq; use sea_orm::prelude::*; use thiserror::Error; @@ -248,11 +249,18 @@ impl ChartObject { T: ChartProperties + QuerySerialized + Send + 'static, QuerySerializedDyn: Into, { + let type_specifics = as Into>::into( + std::sync::Arc::new(Box::new(t)), + ); + assert_eq!( + type_specifics.as_chart_type(), + T::chart_type(), + "data returned by chart {} does not match chart type", + T::name() + ); Self { properties: ChartPropertiesObject::construct_from_chart::(), - type_specifics: as Into>::into( - std::sync::Arc::new(Box::new(t)), - ), + type_specifics, } } } @@ -265,7 +273,6 @@ pub struct ChartPropertiesObject { /// unique identifier of the chart pub key: ChartKey, pub name: String, - pub chart_type: ChartType, pub resolution: ResolutionKind, pub missing_date_policy: MissingDatePolicy, pub approximate_trailing_points: u64, @@ -276,7 +283,6 @@ impl ChartPropertiesObject { Self { key: T::key(), name: T::name(), - chart_type: T::chart_type(), resolution: T::resolution(), missing_date_policy: T::missing_date_policy(), approximate_trailing_points: T::approximate_trailing_points(), diff --git a/stats/stats/src/charts/query_dispatch.rs b/stats/stats/src/charts/query_dispatch.rs index a13085e44..5ad3f5fa1 100644 --- a/stats/stats/src/charts/query_dispatch.rs +++ b/stats/stats/src/charts/query_dispatch.rs @@ -1,6 +1,7 @@ use std::{fmt::Debug, future::Future, ops::Range, sync::Arc}; use chrono::{DateTime, NaiveDate, Utc}; +use entity::sea_orm_active_enums::ChartType; use stats_proto::blockscout::stats::v1::Point; use crate::{ @@ -54,6 +55,15 @@ impl Debug for ChartTypeSpecifics { } } +impl ChartTypeSpecifics { + pub fn as_chart_type(&self) -> ChartType { + match self { + Self::Counter { query: _ } => ChartType::Counter, + Self::Line { query: _ } => ChartType::Line, + } + } +} + impl Into for QuerySerializedDyn> { fn into(self) -> ChartTypeSpecifics { ChartTypeSpecifics::Counter { query: self } From b4c0ab81e91bf0137e4f9e38e72f6e3289a6735e Mon Sep 17 00:00:00 2001 From: Kirill Ivanov Date: Thu, 28 Nov 2024 14:40:59 +0900 Subject: [PATCH 09/41] use chart objects to read counters in read service --- stats/stats-server/src/read_service.rs | 131 +++++++++++++----- stats/stats-server/src/server.rs | 5 +- .../stats-server/tests/it/indexing_status.rs | 6 +- stats/stats/src/charts/chart.rs | 1 - stats/stats/src/charts/query_dispatch.rs | 9 +- stats/stats/src/data_source/types.rs | 1 + 6 files changed, 106 insertions(+), 47 deletions(-) diff --git a/stats/stats-server/src/read_service.rs b/stats/stats-server/src/read_service.rs index 6e513cc10..96ca41fe9 100644 --- a/stats/stats-server/src/read_service.rs +++ b/stats/stats-server/src/read_service.rs @@ -1,21 +1,27 @@ use std::{clone::Clone, cmp::Ord, collections::BTreeMap, fmt::Debug, str::FromStr, sync::Arc}; use crate::{ - config::{layout::sorted_items_according_to_layout, types}, + config::{ + layout::sorted_items_according_to_layout, + types::{self, EnabledChartSettings}, + }, runtime_setup::{EnabledChartEntry, RuntimeSetup}, settings::LimitsSettings, }; +use anyhow::Context; use async_trait::async_trait; -use chrono::{NaiveDate, Utc}; +use chrono::{DateTime, NaiveDate, Utc}; +use futures::{stream::FuturesOrdered, StreamExt}; use proto_v1::stats_service_server::StatsService; use sea_orm::{DatabaseConnection, DbErr}; use stats::{ + data_source::{types::BlockscoutMigrations, UpdateContext, UpdateParameters}, entity::sea_orm_active_enums::ChartType, - query_dispatch::serialize_line_points, + query_dispatch::{serialize_line_points, ChartTypeSpecifics, QuerySerializedDyn}, types::{ timespans::{Month, Week, Year}, - Timespan, + Timespan, TimespanValue, }, ApproxUnsignedDiff, MissingDatePolicy, ReadError, RequestedPointsLimit, ResolutionKind, }; @@ -25,6 +31,7 @@ use tonic::{Request, Response, Status}; #[derive(Clone)] pub struct ReadService { db: Arc, + blockscout: Arc, charts: Arc, limits: ReadLimits, } @@ -32,10 +39,16 @@ pub struct ReadService { impl ReadService { pub async fn new( db: Arc, + blockscout: Arc, charts: Arc, limits: ReadLimits, ) -> Result { - Ok(Self { db, charts, limits }) + Ok(Self { + db, + blockscout, + charts, + limits, + }) } } @@ -178,51 +191,93 @@ async fn get_serialized_line_chart_data_resolution_dispatch( } } +impl ReadService { + async fn query_with_handle( + &self, + query_handle: QuerySerializedDyn, + query_time: DateTime, + ) -> anyhow::Result { + let migrations = BlockscoutMigrations::query_from_db(&self.blockscout) + .await + .context("querying migrations from db")?; + let context = UpdateContext::from_params_now_or_override(UpdateParameters { + db: &self.db, + blockscout: &self.blockscout, + blockscout_applied_migrations: migrations, + update_time_override: Some(query_time), + force_full: false, + }); + query_handle + .query_data(&context, None, true) + .await + .context("querying data") + } + + async fn query_counter_with_handle( + &self, + name: &str, + settings: EnabledChartSettings, + query_handle: QuerySerializedDyn>, + query_time: DateTime, + ) -> anyhow::Result { + let point = self.query_with_handle(query_handle, query_time).await?; + Ok(proto_v1::Counter { + id: name.to_string(), + value: point.value, + title: settings.title, + description: settings.description, + units: settings.units, + }) + } +} + #[async_trait] impl StatsService for ReadService { async fn get_counters( &self, _request: Request, ) -> Result, Status> { - let mut data = stats::get_raw_counters(&self.db) - .await - .map_err(map_read_error)?; - - let counters = self + let now = Utc::now(); + let counters_futures: FuturesOrdered<_> = self .charts .charts_info .iter() - .filter(|(_, chart)| { - chart.resolutions.iter().all(|(_, static_info)| { - static_info.type_specifics.as_chart_type() == ChartType::Counter - }) - }) .filter_map(|(name, counter)| { - data.remove(name).and_then(|point| { - // resolutions other than day are currently not supported - // for counters - let Some(static_info) = counter.resolutions.get(&ResolutionKind::Day) else { - tracing::warn!( - "No 'day' resolution enabled for counter {}, skipping its value", - name - ); - return None; - }; - let point = if static_info.missing_date_policy == MissingDatePolicy::FillZero { - point.relevant_or_zero(Utc::now().date_naive()) - } else { - point - }; - Some(proto_v1::Counter { - id: static_info.name.clone(), - value: point.value, - title: counter.settings.title.clone(), - description: counter.settings.description.clone(), - units: counter.settings.units.clone(), - }) - }) + // resolutions other than day are currently not supported + // for counters + let Some(enabled_resolution) = counter.resolutions.get(&ResolutionKind::Day) else { + tracing::warn!( + "No 'day' resolution enabled for counter {}, skipping its value", + name + ); + return None; + }; + let ChartTypeSpecifics::Counter { + query: query_handle, + } = &enabled_resolution.type_specifics + else { + return None; + }; + Some(self.query_counter_with_handle( + name, + counter.settings.clone(), + query_handle.clone(), + now.clone(), + )) }) .collect(); + let counters: Vec = counters_futures + .filter_map(|query_result| async move { + match query_result { + Ok(c) => Some(c), + Err(e) => { + tracing::error!("Failed to query counter: {:?}", e); + None + } + } + }) + .collect() + .await; let counters_sorted = sorted_items_according_to_layout(counters, &self.charts.counters_layout, |c| &c.id); let counters = proto_v1::Counters { diff --git a/stats/stats-server/src/server.rs b/stats/stats-server/src/server.rs index 0bd45a44f..f0a1da110 100644 --- a/stats/stats-server/src/server.rs +++ b/stats/stats-server/src/server.rs @@ -108,7 +108,7 @@ pub async fn stats(mut settings: Settings) -> Result<(), anyhow::Error> { let blockscout_api_config = init_blockscout_api_client(&settings).await?; let update_service = - Arc::new(UpdateService::new(db.clone(), blockscout, charts.clone()).await?); + Arc::new(UpdateService::new(db.clone(), blockscout.clone(), charts.clone()).await?); let update_service_handle = tokio::spawn(async move { // Wait for blockscout to index, if necessary. @@ -135,7 +135,8 @@ pub async fn stats(mut settings: Settings) -> Result<(), anyhow::Error> { metrics::initialize_metrics(charts.charts_info.keys().map(|f| f.as_str())); } - let read_service = Arc::new(ReadService::new(db, charts, settings.limits.into()).await?); + let read_service = + Arc::new(ReadService::new(db, blockscout, charts, settings.limits.into()).await?); let health = Arc::new(HealthService::default()); let grpc_router = grpc_router(read_service.clone(), health.clone()); diff --git a/stats/stats-server/tests/it/indexing_status.rs b/stats/stats-server/tests/it/indexing_status.rs index d69b072f8..ba6be3887 100644 --- a/stats/stats-server/tests/it/indexing_status.rs +++ b/stats/stats-server/tests/it/indexing_status.rs @@ -85,6 +85,8 @@ async fn test_not_indexed_ok() { } } - let counters: Counters = send_get_request(&base, "/api/v1/counters").await; - assert!(counters.counters.is_empty()) + let mut counters: Counters = send_get_request(&base, "/api/v1/counters").await; + // totalBlocks has fallback with estimate, so it should always return + assert_eq!(counters.counters.pop().unwrap().id, "totalBlocks"); + assert_eq!(counters.counters, vec![]) } diff --git a/stats/stats/src/charts/chart.rs b/stats/stats/src/charts/chart.rs index 4bca1bf7a..52e2e4457 100644 --- a/stats/stats/src/charts/chart.rs +++ b/stats/stats/src/charts/chart.rs @@ -9,7 +9,6 @@ use std::fmt::Display; use crate::{types::Timespan, ReadError}; use chrono::{DateTime, Utc}; use entity::sea_orm_active_enums::{ChartResolution, ChartType}; -use pretty_assertions::assert_eq; use sea_orm::prelude::*; use thiserror::Error; diff --git a/stats/stats/src/charts/query_dispatch.rs b/stats/stats/src/charts/query_dispatch.rs index 5ad3f5fa1..5b5adc27f 100644 --- a/stats/stats/src/charts/query_dispatch.rs +++ b/stats/stats/src/charts/query_dispatch.rs @@ -1,4 +1,4 @@ -use std::{fmt::Debug, future::Future, ops::Range, sync::Arc}; +use std::{fmt::Debug, future::Future, ops::Range, pin::Pin, sync::Arc}; use chrono::{DateTime, NaiveDate, Utc}; use entity::sea_orm_active_enums::ChartType; @@ -31,7 +31,7 @@ pub trait QuerySerialized { cx: &UpdateContext<'a>, range: Option>>, fill_missing_dates: bool, - ) -> Box> + Send + 'a>; + ) -> Pin> + Send + 'a>>; } /// [`QuerySerialized`] but for dynamic dispatch @@ -118,9 +118,10 @@ where cx: &UpdateContext<'a>, range: Option>>, fill_missing_dates: bool, - ) -> Box> + Send + 'a> { + ) -> Pin> + Send + 'a>> + { let cx = cx.clone(); - Box::new(async move { + Box::pin(async move { let data = Query::query_data(&cx, range, fill_missing_dates).await?; Ok(data.serialize()) }) diff --git a/stats/stats/src/data_source/types.rs b/stats/stats/src/data_source/types.rs index 56e3b3486..6f9ae5c5e 100644 --- a/stats/stats/src/data_source/types.rs +++ b/stats/stats/src/data_source/types.rs @@ -1,3 +1,4 @@ +use anyhow::Context; use blockscout_db::entity::migrations_status; use chrono::Utc; use sea_orm::{DatabaseConnection, DbErr, EntityTrait, FromQueryResult, QueryOrder, Statement}; From 7ef2ad2b84526a4a51b4d26fb68848e2f5bc5a63 Mon Sep 17 00:00:00 2001 From: Kirill Ivanov Date: Thu, 28 Nov 2024 15:37:35 +0900 Subject: [PATCH 10/41] dynamic query dispatch for line charts WIP --- stats/stats-server/src/read_service.rs | 45 ++++++++++++++++-------- stats/stats/src/charts/query_dispatch.rs | 42 +++++++++++++++------- 2 files changed, 61 insertions(+), 26 deletions(-) diff --git a/stats/stats-server/src/read_service.rs b/stats/stats-server/src/read_service.rs index 96ca41fe9..b916f15df 100644 --- a/stats/stats-server/src/read_service.rs +++ b/stats/stats-server/src/read_service.rs @@ -252,16 +252,14 @@ impl StatsService for ReadService { ); return None; }; - let ChartTypeSpecifics::Counter { - query: query_handle, - } = &enabled_resolution.type_specifics - else { - return None; - }; + let query_handle = enabled_resolution + .type_specifics + .clone() + .into_counter_handle()?; Some(self.query_counter_with_handle( name, counter.settings.clone(), - query_handle.clone(), + query_handle, now.clone(), )) }) @@ -296,13 +294,22 @@ impl StatsService for ReadService { let chart_entry = self.charts.charts_info.get(&chart_name).ok_or_else(|| { Status::not_found(format!("chart with name '{}' was not found", chart_name)) })?; - let resolution_info = chart_entry - .resolutions - .get(&resolution) - .filter(|static_info| static_info.type_specifics.as_chart_type() == ChartType::Line) + let resolution_info = chart_entry.resolutions.get(&resolution); + let (query_handle, policy, mark_approx) = resolution_info + .and_then(|enabled_resolution| { + let handle = enabled_resolution + .type_specifics + .clone() + .into_line_handle()?; + Some(( + handle, + enabled_resolution.missing_date_policy, + enabled_resolution.approximate_trailing_points, + )) + }) .ok_or_else(|| { Status::not_found(format!( - "resolution '{}' for chart '{}' was not found", + "resolution '{}' for line chart '{}' was not found", String::from(resolution), chart_name, )) @@ -312,9 +319,19 @@ impl StatsService for ReadService { .from .and_then(|date| NaiveDate::from_str(&date).ok()); let to = request.to.and_then(|date| NaiveDate::from_str(&date).ok()); - let policy = resolution_info.missing_date_policy; - let mark_approx = resolution_info.approximate_trailing_points; let points_limit = Some(self.limits.requested_points_limit); + + let migrations = BlockscoutMigrations::query_from_db(&self.blockscout) + .await + .map_err(|e| map_read_error(ReadError::DB(e)))?; + let context = UpdateContext::from_params_now_or_override(UpdateParameters { + db: &self.db, + blockscout: &self.blockscout, + blockscout_applied_migrations: migrations, + update_time_override: Some(Utc::now()), + force_full: false, + }); + let data = query_handle.query_data().await.map_err(map_read_error)?; let serialized_chart = get_serialized_line_chart_data_resolution_dispatch( &self.db, chart_name.clone(), diff --git a/stats/stats/src/charts/query_dispatch.rs b/stats/stats/src/charts/query_dispatch.rs index 5b5adc27f..e90b8ad8b 100644 --- a/stats/stats/src/charts/query_dispatch.rs +++ b/stats/stats/src/charts/query_dispatch.rs @@ -12,7 +12,7 @@ use crate::{ }, DataSource, UpdateContext, }, - exclusive_datetime_range_to_inclusive, + exclusive_datetime_range_to_inclusive, RequestedPointsLimit, }; use super::{ @@ -29,7 +29,9 @@ pub trait QuerySerialized { fn query_data<'a>( &self, cx: &UpdateContext<'a>, - range: Option>>, + from: Option>, + to: Option>, + points_limit: Option, fill_missing_dates: bool, ) -> Pin> + Send + 'a>>; } @@ -37,13 +39,13 @@ pub trait QuerySerialized { /// [`QuerySerialized`] but for dynamic dispatch pub type QuerySerializedDyn = Arc + Send + Sync>>; +pub type CounterHandle = QuerySerializedDyn>; +pub type LineHandle = QuerySerializedDyn>; + +#[derive(Clone)] pub enum ChartTypeSpecifics { - Counter { - query: QuerySerializedDyn>, - }, - Line { - query: QuerySerializedDyn>, - }, + Counter { query: CounterHandle }, + Line { query: LineHandle }, } impl Debug for ChartTypeSpecifics { @@ -62,15 +64,29 @@ impl ChartTypeSpecifics { Self::Line { query: _ } => ChartType::Line, } } + + pub fn into_counter_handle(self) -> Option { + match self { + Self::Counter { query } => Some(query), + _ => None, + } + } + + pub fn into_line_handle(self) -> Option { + match self { + Self::Line { query } => Some(query), + _ => None, + } + } } -impl Into for QuerySerializedDyn> { +impl Into for CounterHandle { fn into(self) -> ChartTypeSpecifics { ChartTypeSpecifics::Counter { query: self } } } -impl Into for QuerySerializedDyn> { +impl Into for LineHandle { fn into(self) -> ChartTypeSpecifics { ChartTypeSpecifics::Line { query: self } } @@ -116,13 +132,15 @@ where fn query_data<'a>( &self, cx: &UpdateContext<'a>, - range: Option>>, + from: Option>, + to: Option>, + points_limit: Option, fill_missing_dates: bool, ) -> Pin> + Send + 'a>> { let cx = cx.clone(); Box::pin(async move { - let data = Query::query_data(&cx, range, fill_missing_dates).await?; + let data = Query::query_data(&cx, range, points_limit, fill_missing_dates).await?; Ok(data.serialize()) }) } From 68aad38aef3e9e90ffe4eb31a720541bed35bb51 Mon Sep 17 00:00:00 2001 From: Kirill Ivanov Date: Mon, 2 Dec 2024 14:53:10 +0900 Subject: [PATCH 11/41] timespan checked operations + max/min --- stats/stats/src/charts/types/timespans/day.rs | 14 ++++-- .../stats/src/charts/types/timespans/month.rs | 24 +++++----- .../stats/src/charts/types/timespans/week.rs | 24 +++++----- .../stats/src/charts/types/timespans/year.rs | 44 +++++++++++++++++-- stats/stats/src/charts/types/traits.rs | 18 +++++++- 5 files changed, 95 insertions(+), 29 deletions(-) diff --git a/stats/stats/src/charts/types/timespans/day.rs b/stats/stats/src/charts/types/timespans/day.rs index b3544a9a7..4c02f4408 100644 --- a/stats/stats/src/charts/types/timespans/day.rs +++ b/stats/stats/src/charts/types/timespans/day.rs @@ -37,20 +37,26 @@ impl Timespan for NaiveDate { day_start(self) } - fn saturating_add(&self, duration: TimespanDuration) -> Self + fn checked_add(&self, duration: TimespanDuration) -> Option where Self: Sized, { self.checked_add_days(Days::new(duration.repeats())) - .unwrap_or(NaiveDate::MAX) } - fn saturating_sub(&self, duration: TimespanDuration) -> Self + fn checked_sub(&self, duration: TimespanDuration) -> Option where Self: Sized, { self.checked_sub_days(Days::new(duration.repeats())) - .unwrap_or(NaiveDate::MIN) + } + + fn max() -> Self { + NaiveDate::MAX + } + + fn min() -> Self { + NaiveDate::MIN } } diff --git a/stats/stats/src/charts/types/timespans/month.rs b/stats/stats/src/charts/types/timespans/month.rs index df1e2bf9c..4c7cbe867 100644 --- a/stats/stats/src/charts/types/timespans/month.rs +++ b/stats/stats/src/charts/types/timespans/month.rs @@ -73,30 +73,34 @@ impl Timespan for Month { self.saturating_first_day().saturating_start_timestamp() } - fn saturating_add(&self, duration: TimespanDuration) -> Self + fn checked_add(&self, duration: TimespanDuration) -> Option where Self: Sized, { - let result_month_date = self - .date_in_month + self.date_in_month .checked_add_months(chrono::Months::new( duration.repeats().try_into().unwrap_or(u32::MAX), )) - .unwrap_or(NaiveDate::MAX); - Self::from_date(result_month_date) + .map(Self::from_date) } - fn saturating_sub(&self, duration: TimespanDuration) -> Self + fn checked_sub(&self, duration: TimespanDuration) -> Option where Self: Sized, { - let result_month_date = self - .date_in_month + self.date_in_month .checked_sub_months(chrono::Months::new( duration.repeats().try_into().unwrap_or(u32::MAX), )) - .unwrap_or(NaiveDate::MIN); - Self::from_date(result_month_date) + .map(Self::from_date) + } + + fn max() -> Self { + Self::from_date(NaiveDate::MAX) + } + + fn min() -> Self { + Self::from_date(NaiveDate::MIN) } } diff --git a/stats/stats/src/charts/types/timespans/week.rs b/stats/stats/src/charts/types/timespans/week.rs index 3ddf83c56..e27a7e1e8 100644 --- a/stats/stats/src/charts/types/timespans/week.rs +++ b/stats/stats/src/charts/types/timespans/week.rs @@ -88,26 +88,30 @@ impl Timespan for Week { self.saturating_first_day().saturating_start_timestamp() } - fn saturating_add(&self, duration: TimespanDuration) -> Self + fn checked_add(&self, duration: TimespanDuration) -> Option where Self: Sized, { - let result_week_date = self - .saturating_first_day() + self.saturating_first_day() .checked_add_days(Days::new(duration.repeats() * 7)) - .unwrap_or(NaiveDate::MAX); - Self::from_date(result_week_date) + .map(Self::from_date) } - fn saturating_sub(&self, duration: TimespanDuration) -> Self + fn checked_sub(&self, duration: TimespanDuration) -> Option where Self: Sized, { - let result_week_date = self - .saturating_first_day() + self.saturating_first_day() .checked_sub_days(Days::new(duration.repeats() * 7)) - .unwrap_or(NaiveDate::MIN); - Self::from_date(result_week_date) + .map(Self::from_date) + } + + fn max() -> Self { + Self::from_date(NaiveDate::MAX) + } + + fn min() -> Self { + Self::from_date(NaiveDate::MIN) } } diff --git a/stats/stats/src/charts/types/timespans/year.rs b/stats/stats/src/charts/types/timespans/year.rs index 793cf4b49..cca7565bf 100644 --- a/stats/stats/src/charts/types/timespans/year.rs +++ b/stats/stats/src/charts/types/timespans/year.rs @@ -55,13 +55,17 @@ impl Year { self.clamp_by_naive_date_range().0 } + fn year_number_within_naive_date(year: i32) -> bool { + NaiveDate::from_yo_opt(year, 1).is_some() + } + fn clamp_by_naive_date_range(self) -> Self { - if NaiveDate::from_yo_opt(self.0, 1).is_some() { + if Self::year_number_within_naive_date(self.0) { self } else if self.0 > 0 { - Self::from_date(NaiveDate::MAX) + ::max() } else { - Self::from_date(NaiveDate::MIN) + ::min() } } } @@ -98,6 +102,40 @@ impl Timespan for Year { let sub_years: i32 = duration.repeats().try_into().unwrap_or(i32::MAX); Self(self.number_within_naive_date().saturating_sub(sub_years)).clamp_by_naive_date_range() } + + fn checked_add(&self, duration: TimespanDuration) -> Option + where + Self: Sized, + { + let add_years = duration.repeats().try_into().unwrap_or(i32::MAX); + let new_year_num = self.number_within_naive_date().checked_add(add_years)?; + if Self::year_number_within_naive_date(new_year_num) { + Some(Self(new_year_num)) + } else { + None + } + } + + fn checked_sub(&self, duration: TimespanDuration) -> Option + where + Self: Sized, + { + let sub_years: i32 = duration.repeats().try_into().unwrap_or(i32::MAX); + let new_year_num = self.number_within_naive_date().checked_sub(sub_years)?; + if Self::year_number_within_naive_date(new_year_num) { + Some(Self(new_year_num)) + } else { + None + } + } + + fn max() -> Self { + Self::from_date(NaiveDate::MAX) + } + + fn min() -> Self { + Self::from_date(NaiveDate::MIN) + } } impl ConsistsOf for Year { diff --git a/stats/stats/src/charts/types/traits.rs b/stats/stats/src/charts/types/traits.rs index 1afe9b83a..82a795982 100644 --- a/stats/stats/src/charts/types/traits.rs +++ b/stats/stats/src/charts/types/traits.rs @@ -44,12 +44,26 @@ pub trait Timespan { self.saturating_start_timestamp() ..self.saturating_next_timespan().saturating_start_timestamp() } - fn saturating_add(&self, duration: TimespanDuration) -> Self + fn checked_add(&self, duration: TimespanDuration) -> Option where Self: Sized; - fn saturating_sub(&self, duration: TimespanDuration) -> Self + fn checked_sub(&self, duration: TimespanDuration) -> Option where Self: Sized; + fn max() -> Self; + fn min() -> Self; + fn saturating_add(&self, duration: TimespanDuration) -> Self + where + Self: Sized, + { + self.checked_add(duration).unwrap_or_else(|| Self::max()) + } + fn saturating_sub(&self, duration: TimespanDuration) -> Self + where + Self: Sized, + { + self.checked_sub(duration).unwrap_or_else(|| Self::min()) + } } // if for some rare reason trait is needed From 59a9d081e39d438bcc50311501f45b106a6aa870 Mon Sep 17 00:00:00 2001 From: Kirill Ivanov Date: Mon, 2 Dec 2024 21:29:25 +0900 Subject: [PATCH 12/41] new universal range type --- stats/stats/src/range.rs | 493 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 493 insertions(+) create mode 100644 stats/stats/src/range.rs diff --git a/stats/stats/src/range.rs b/stats/stats/src/range.rs new file mode 100644 index 000000000..4156ceb68 --- /dev/null +++ b/stats/stats/src/range.rs @@ -0,0 +1,493 @@ +use std::ops::{Bound, Range, RangeBounds, RangeInclusive}; + +use chrono::{DateTime, Utc}; + +use crate::{ + data_source::{kinds::remote_db::RemoteQueryBehaviour, UpdateContext}, + types::{Timespan, TimespanDuration}, + UpdateError, +}; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct UniversalRange { + /// Always inclusive, if present + pub start: Option, + pub end: Bound, +} + +impl UniversalRange { + pub fn full() -> Self { + Self { + start: None, + end: Bound::Unbounded, + } + } + + pub fn with_replaced_unbounded(self, replacement_source: Range) -> Self { + let start = match self.start { + Some(s) => Some(s), + None => Some(replacement_source.start), + }; + let end = match self.end { + Bound::Unbounded => Bound::Excluded(replacement_source.end), + _ => self.end, + }; + Self { start, end } + } + + pub fn map(self, f: impl Fn(Idx) -> T) -> UniversalRange { + UniversalRange { + start: self.start.map(&f), + end: self.end.map(f), + } + } +} + +impl From> for UniversalRange { + fn from(value: Range) -> Self { + Self { + start: Some(value.start), + end: Bound::Excluded(value.end), + } + } +} + +impl From>> for UniversalRange { + fn from(value: Range>) -> Self { + Self { + start: value.start, + end: match value.end { + Some(e) => Bound::Excluded(e), + None => Bound::Unbounded, + }, + } + } +} + +impl From> for UniversalRange { + fn from(value: RangeInclusive) -> Self { + let (start, end) = value.into_inner(); + Self { + start: Some(start), + end: Bound::Included(end), + } + } +} + +impl From>> for UniversalRange { + fn from(value: RangeInclusive>) -> Self { + let (start, end) = value.into_inner(); + Self { + start: start, + end: match end { + Some(e) => Bound::Included(e), + None => Bound::Unbounded, + }, + } + } +} + +impl> UniversalRange { + pub fn is_unbounded(&self) -> bool { + self.start.is_none() || self.end == Bound::Unbounded + } +} + +impl UniversalRange { + /// None only if no backup is provided + fn into_exclusive_inner(self, backup_bounds: Option>) -> Option> { + let (backup_start, backup_end) = backup_bounds.map(|b| (b.start, b.end)).unzip(); + // todo: test that none backup and some start works as expected (+ end) + let start = self.start.or(backup_start)?; + match self.end { + Bound::Included(end) => Some(inclusive_range_to_exclusive(start..=end)), + Bound::Excluded(end) => Some(start..end), + Bound::Unbounded => { + let end = backup_end?; + Some(start..end) + } + } + } + + /// `None` if any of the ends is unbounded + pub fn try_into_exclusive(self) -> Option> { + self.into_exclusive_inner(None) + } + + /// Take bounds from `backup_bounds` if any of them is unbounded. + pub fn into_exclusive_with_backup(self, backup_bounds: Range) -> Range { + self.into_exclusive_inner(Some(backup_bounds)) + .expect("`backup_bounds` is not None") + } +} + +impl UniversalRange { + fn into_inclusive_inner( + self, + backup_bounds: Option>, + ) -> Option> { + let (backup_start, backup_end) = backup_bounds.map(|b| b.into_inner()).unzip(); + let start = self.start.or(backup_start)?; + match self.end { + Bound::Included(end) => Some(start..=end), + Bound::Excluded(end) => Some(exclusive_range_to_inclusive(start..end)), + Bound::Unbounded => { + let end = backup_end?; + Some(start..=end) + } + } + } + + /// `None` if any of the ends is unbounded + pub fn try_into_inclusive(self) -> Option> { + self.into_inclusive_inner(None) + } + + /// Take bounds from `backup_bounds` if any of them is unbounded. + pub fn into_inclusive_with_backup( + self, + backup_bounds: RangeInclusive, + ) -> RangeInclusive { + self.into_inclusive_inner(Some(backup_bounds)) + .expect("`backup_bounds` is not None") + } + + pub fn into_inclusive_pair(self) -> (Option, Option) { + match (self.start, self.end) { + (start, Bound::Unbounded) => (start, None), + (start, Bound::Included(e)) => (start, Some(e)), + (None, Bound::Excluded(e)) => (None, Some(e.saturating_dec())), + (Some(s), Bound::Excluded(e)) => { + Some(exclusive_range_to_inclusive(s..e).into_inner()).unzip() + } + } + } +} + +impl> RangeBounds for UniversalRange { + fn start_bound(&self) -> Bound<&Idx> { + match &self.start { + Some(s) => Bound::Included(s), + None => Bound::Unbounded, + } + } + + fn end_bound(&self) -> Bound<&Idx> { + self.end.as_ref() + } +} + +pub fn exclusive_range_to_inclusive( + r: Range, +) -> RangeInclusive { + let mut start = r.start; + let end = match r.end.checked_dec() { + Some(new_end) => new_end, + None => { + // current end is the minimum value and is excluded, thus + // `self` range is empty + // so we need to produce an empty range as well + if start.checked_dec().is_none() { + // will not produce an empty range iff there is only + // one value of `Idx` type, which will not be encountered + // in practice + start = start.saturating_inc(); + } + r.end + } + }; + start..=end +} + +pub fn inclusive_range_to_exclusive(r: RangeInclusive) -> Range { + let (start, end) = r.into_inner(); + // impossible to include max value in exclusive range, + // so we leave it excluded in such case + start..end.saturating_inc() +} + +pub trait Incrementable { + fn saturating_inc(&self) -> Self; + fn checked_inc(&self) -> Option + where + Self: Sized; +} + +impl Incrementable for T { + fn saturating_inc(&self) -> Self { + self.saturating_next_timespan() + } + + fn checked_inc(&self) -> Option + where + Self: Sized, + { + self.checked_add(TimespanDuration::from_timespan_repeats(1)) + } +} + +impl Incrementable for DateTime { + fn saturating_inc(&self) -> Self { + self.checked_inc().unwrap_or(DateTime::::MAX_UTC) + } + + fn checked_inc(&self) -> Option { + self.checked_add_signed(chrono::Duration::nanoseconds(1)) + } +} + +// for tests +impl Incrementable for i32 { + fn saturating_inc(&self) -> Self { + self.saturating_add(1) + } + + fn checked_inc(&self) -> Option { + self.checked_add(1) + } +} + +pub trait Decrementable { + fn saturating_dec(&self) -> Self; + fn checked_dec(&self) -> Option + where + Self: Sized; +} + +impl Decrementable for T { + fn saturating_dec(&self) -> Self { + self.saturating_previous_timespan() + } + + fn checked_dec(&self) -> Option + where + Self: Sized, + { + self.checked_sub(TimespanDuration::from_timespan_repeats(1)) + } +} + +impl Decrementable for DateTime { + fn saturating_dec(&self) -> Self { + self.checked_dec().unwrap_or(DateTime::::MIN_UTC) + } + + fn checked_dec(&self) -> Option { + self.checked_sub_signed(chrono::Duration::nanoseconds(1)) + } +} + +impl Decrementable for i32 { + fn saturating_dec(&self) -> Self { + self.saturating_sub(1) + } + + fn checked_dec(&self) -> Option { + self.checked_sub(1) + } +} + +// todo (future releases/features): implement one-sided range support +// for db statements (in `filter` part) +/// DB statements have a regular range to keep their implementation simpler +/// but data sources have flexible bound setting. This function converts +/// from one to another while ensuring that non-range queries stay the same. +/// +/// At the same time, it adds support for one-side unbounded ranges by +/// quering the provided all range source in such case. +pub async fn data_source_query_range_to_db_statement_range( + cx: &UpdateContext<'_>, + data_source_range: UniversalRange>, +) -> Result>>, UpdateError> +where + AllRangeSource: RemoteQueryBehaviour>>, +{ + let range = if let Some(r) = data_source_range.clone().try_into_exclusive() { + Some(r) + } else if data_source_range.start.is_none() && data_source_range.end == Bound::Unbounded { + None + } else { + let whole_range = AllRangeSource::query_data(cx, UniversalRange::full()).await?; + Some(data_source_range.into_exclusive_with_backup(whole_range)) + }; + Ok(range) +} +#[cfg(test)] +mod tests { + use super::*; + use crate::{tests::point_construction::*, types::timespans::Week}; + use chrono::{NaiveDate, NaiveDateTime}; + use pretty_assertions::assert_eq; + + #[test] + fn test_with_replaced_unbounded() { + let range = UniversalRange { + start: None, + end: Bound::Unbounded, + }; + let replacement = dt("2023-01-01T00:00:00")..dt("2023-01-02T00:00:00"); + let result = range.with_replaced_unbounded(replacement); + assert_eq!(result.start, Some(dt("2023-01-01T00:00:00"))); + assert_eq!(result.end, Bound::Excluded(dt("2023-01-02T00:00:00"))); + + let partial_range = UniversalRange { + start: Some(dt("2023-01-01T12:00:00")), + end: Bound::Unbounded, + }; + let result = partial_range + .with_replaced_unbounded(dt("2023-01-01T00:00:00")..dt("2023-01-02T00:00:00")); + assert_eq!(result.start, Some(dt("2023-01-01T12:00:00"))); + assert_eq!(result.end, Bound::Excluded(dt("2023-01-02T00:00:00"))); + } + + #[test] + fn test_conversion_functions() { + // Test with weeks + let exclusive = d("2023-01-01")..d("2023-01-15"); + let inclusive = exclusive_range_to_inclusive(exclusive); + assert_eq!(inclusive, d("2023-01-01")..=d("2023-01-14")); + + let inclusive = d("2023-01-01")..=d("2023-01-14"); + let exclusive = inclusive_range_to_exclusive(inclusive); + assert_eq!(exclusive, d("2023-01-01")..d("2023-01-15")); + + // Test with months + let exclusive = month_of("2023-01-01")..month_of("2023-03-01"); + let inclusive = exclusive_range_to_inclusive(exclusive); + assert_eq!(inclusive, month_of("2023-01-01")..=month_of("2023-02-01")); + } + + #[test] + fn test_into_exclusive_conversions() { + let range: UniversalRange = (d("2023-01-01")..d("2023-01-15")).into(); + assert_eq!( + range.clone().try_into_exclusive(), + Some(d("2023-01-01")..d("2023-01-15")) + ); + + let inclusive: UniversalRange = (d("2023-01-01")..=d("2023-01-14")).into(); + assert_eq!( + inclusive.clone().try_into_exclusive(), + Some(d("2023-01-01")..d("2023-01-15")) + ); + + let unbounded = UniversalRange::full(); + assert_eq!(unbounded.clone().try_into_exclusive(), None); + + let with_backup = unbounded.into_exclusive_with_backup(d("2023-01-01")..d("2023-01-15")); + assert_eq!(with_backup, d("2023-01-01")..d("2023-01-15")); + } + + #[test] + fn test_into_inclusive_conversions() { + let range: UniversalRange = (d("2023-01-01")..=d("2023-01-15")).into(); + assert_eq!( + range.clone().try_into_inclusive(), + Some(d("2023-01-01")..=d("2023-01-15")) + ); + + let exclusive: UniversalRange = (d("2023-01-01")..d("2023-01-15")).into(); + assert_eq!( + exclusive.clone().try_into_inclusive(), + Some(d("2023-01-01")..=d("2023-01-14")) + ); + + let unbounded = UniversalRange::full(); + assert_eq!(unbounded.clone().try_into_inclusive(), None); + + let with_backup = unbounded.into_inclusive_with_backup(d("2023-01-01")..=d("2023-01-08")); + assert_eq!(with_backup, d("2023-01-01")..=d("2023-01-08")); + } + + #[test] + fn test_into_inclusive_pair() { + // Basic inclusive range + let inclusive: UniversalRange = (1..=5).into(); + assert_eq!(inclusive.into_inclusive_pair(), (Some(1), Some(5))); + + // Basic exclusive range + let exclusive: UniversalRange = (1..6).into(); + assert_eq!(exclusive.into_inclusive_pair(), (Some(1), Some(5))); + + // Unbounded range + let unbounded = UniversalRange::::full(); + assert_eq!(unbounded.into_inclusive_pair(), (None, None)); + + // Start-only range + let start_only = UniversalRange { + start: Some(1), + end: Bound::Unbounded, + }; + assert_eq!(start_only.into_inclusive_pair(), (Some(1), None)); + + // End-only range (exclusive) + let end_only = UniversalRange { + start: None, + end: Bound::Excluded(5), + }; + assert_eq!(end_only.into_inclusive_pair(), (None, Some(4))); + + // End-only range (inclusive) + let end_only_inclusive = UniversalRange { + start: None, + end: Bound::Included(5), + }; + assert_eq!(end_only_inclusive.into_inclusive_pair(), (None, Some(5))); + } + + #[test] + fn test_exclusive_to_inclusive_conversion() { + // Normal case + assert_eq!(exclusive_range_to_inclusive(1..5), 1..=4); + + // Single-element range + assert_eq!(exclusive_range_to_inclusive(1..2), 1..=1); + + // Empty range at start + let empty_start = exclusive_range_to_inclusive(0..0); + assert!(empty_start.is_empty()); + + // Empty range at end of i32 + let empty_end = exclusive_range_to_inclusive(i32::MAX..i32::MAX); + assert!(empty_end.is_empty()); + + // Range ending at i32::MAX + assert_eq!( + exclusive_range_to_inclusive(0..i32::MAX), + 0..=(i32::MAX - 1) + ); + + // Minimal non-empty range + assert_eq!(exclusive_range_to_inclusive(5..6), 5..=5); + } + + #[test] + fn test_inclusive_to_exclusive_conversion() { + // Normal case + assert_eq!(inclusive_range_to_exclusive(1..=4), 1..5); + + // Single element range + assert_eq!(inclusive_range_to_exclusive(1..=1), 1..2); + + // can't represent (i32::MAX..=i32::MAX) as an exclusive range + // so no test for that + + // Range ending at i32::MAX + assert_eq!( + inclusive_range_to_exclusive(0..=i32::MAX), + 0..i32::MAX // Note: this saturates at MAX instead of overflowing + ); + + // Range ending one before MAX + assert_eq!( + inclusive_range_to_exclusive(0..=(i32::MAX - 1)), + 0..i32::MAX + ); + + // Test with minimum values + assert_eq!( + inclusive_range_to_exclusive(i32::MIN..=i32::MIN), + i32::MIN..(i32::MIN + 1) + ); + } +} From f96158a1504383458bba4be0fa53a3f6ae660d38 Mon Sep 17 00:00:00 2001 From: Kirill Ivanov Date: Mon, 2 Dec 2024 21:30:22 +0900 Subject: [PATCH 13/41] dynamic query dispatch for line charts --- stats/stats-server/src/read_service.rs | 213 +++++------------- stats/stats/src/charts/chart.rs | 1 + .../src/charts/counters/average_block_time.rs | 7 +- stats/stats/src/charts/counters/mock.rs | 6 +- .../stats/src/charts/counters/total_blocks.rs | 6 +- .../src/charts/counters/total_contracts.rs | 6 +- .../src/charts/counters/yesterday_txns.rs | 4 +- stats/stats/src/charts/db_interaction/read.rs | 6 +- .../stats/src/charts/lines/active_accounts.rs | 6 +- .../src/charts/lines/average_block_rewards.rs | 6 +- .../src/charts/lines/average_block_size.rs | 6 +- .../src/charts/lines/average_gas_limit.rs | 6 +- .../src/charts/lines/average_gas_price.rs | 6 +- .../stats/src/charts/lines/average_txn_fee.rs | 6 +- .../stats/src/charts/lines/gas_used_growth.rs | 6 +- stats/stats/src/charts/lines/mock.rs | 26 ++- .../src/charts/lines/native_coin_supply.rs | 6 +- stats/stats/src/charts/lines/new_accounts.rs | 18 +- .../src/charts/lines/new_block_rewards.rs | 20 +- stats/stats/src/charts/lines/new_blocks.rs | 6 +- stats/stats/src/charts/lines/new_contracts.rs | 6 +- .../charts/lines/new_native_coin_transfers.rs | 11 +- stats/stats/src/charts/lines/new_txns.rs | 6 +- .../charts/lines/new_verified_contracts.rs | 11 +- stats/stats/src/charts/lines/txns_fee.rs | 5 +- .../src/charts/lines/txns_success_rate.rs | 6 +- stats/stats/src/charts/query_dispatch.rs | 13 +- .../data_source/kinds/auxiliary/cumulative.rs | 5 +- .../kinds/data_manipulation/delta.rs | 22 +- .../data_manipulation/filter_deducible.rs | 26 ++- .../kinds/data_manipulation/last_point.rs | 9 +- .../kinds/data_manipulation/map/mod.rs | 9 +- .../data_manipulation/resolutions/average.rs | 45 ++-- .../resolutions/last_value.rs | 32 +-- .../data_manipulation/resolutions/mod.rs | 73 +++--- .../data_manipulation/resolutions/sum.rs | 36 ++- .../kinds/data_manipulation/sum_point.rs | 10 +- .../src/data_source/kinds/local_db/mod.rs | 12 +- .../kinds/local_db/parameter_traits.rs | 10 +- .../kinds/local_db/parameters/query.rs | 28 ++- .../parameters/update/batching/mod.rs | 13 +- .../kinds/local_db/parameters/update/point.rs | 3 +- .../src/data_source/kinds/remote_db/mod.rs | 6 +- .../data_source/kinds/remote_db/query/all.rs | 24 +- .../data_source/kinds/remote_db/query/each.rs | 11 +- .../data_source/kinds/remote_db/query/one.rs | 11 +- stats/stats/src/data_source/source.rs | 14 +- stats/stats/src/data_source/tests.rs | 6 +- stats/stats/src/data_source/types.rs | 1 - stats/stats/src/lib.rs | 8 +- stats/stats/src/range.rs | 4 +- stats/stats/src/update_group.rs | 7 +- stats/stats/src/utils.rs | 20 +- 53 files changed, 453 insertions(+), 417 deletions(-) diff --git a/stats/stats-server/src/read_service.rs b/stats/stats-server/src/read_service.rs index b916f15df..219c93dfe 100644 --- a/stats/stats-server/src/read_service.rs +++ b/stats/stats-server/src/read_service.rs @@ -1,4 +1,4 @@ -use std::{clone::Clone, cmp::Ord, collections::BTreeMap, fmt::Debug, str::FromStr, sync::Arc}; +use std::{clone::Clone, collections::BTreeMap, fmt::Debug, str::FromStr, sync::Arc}; use crate::{ config::{ @@ -9,7 +9,6 @@ use crate::{ settings::LimitsSettings, }; -use anyhow::Context; use async_trait::async_trait; use chrono::{DateTime, NaiveDate, Utc}; use futures::{stream::FuturesOrdered, StreamExt}; @@ -17,15 +16,13 @@ use proto_v1::stats_service_server::StatsService; use sea_orm::{DatabaseConnection, DbErr}; use stats::{ data_source::{types::BlockscoutMigrations, UpdateContext, UpdateParameters}, - entity::sea_orm_active_enums::ChartType, - query_dispatch::{serialize_line_points, ChartTypeSpecifics, QuerySerializedDyn}, - types::{ - timespans::{Month, Week, Year}, - Timespan, TimespanValue, - }, - ApproxUnsignedDiff, MissingDatePolicy, ReadError, RequestedPointsLimit, ResolutionKind, + query_dispatch::{CounterHandle, LineHandle, QuerySerializedDyn}, + range::UniversalRange, + types::Timespan, + utils::day_start, + RequestedPointsLimit, ResolutionKind, UpdateError, }; -use stats_proto::blockscout::stats::v1::{self as proto_v1, Point}; +use stats_proto::blockscout::stats::v1 as proto_v1; use tonic::{Request, Response, Status}; #[derive(Clone)] @@ -66,10 +63,10 @@ impl From for ReadLimits { } } -fn map_read_error(err: ReadError) -> Status { +fn map_update_error(err: UpdateError) -> Status { match &err { - ReadError::ChartNotFound(_) => Status::not_found(err.to_string()), - ReadError::IntervalTooLarge(_) => Status::invalid_argument(err.to_string()), + UpdateError::ChartNotFound(_) => Status::not_found(err.to_string()), + UpdateError::IntervalTooLarge { limit: _ } => Status::invalid_argument(err.to_string()), _ => { tracing::error!(err = ?err, "internal read error"); Status::internal(err.to_string()) @@ -99,107 +96,17 @@ fn convert_resolution(input: proto_v1::Resolution) -> ResolutionKind { } } -async fn get_serialized_line_chart_data( - db: &DatabaseConnection, - chart_name: String, - from: Option, - to: Option, - points_limit: Option, - policy: MissingDatePolicy, - mark_approx: u64, -) -> Result, ReadError> -where - Resolution: Timespan + ApproxUnsignedDiff + Clone + Ord + Debug, -{ - let from = from.map(|f| Resolution::from_date(f)); - let to = to.map(|t| Resolution::from_date(t)); - let data = stats::get_line_chart_data::( - db, - &chart_name, - from, - to, - points_limit, - policy, - true, - mark_approx, - ) - .await?; - Ok(serialize_line_points(data)) -} - -/// enum dispatch for `get_serialized_line_chart_data` -#[allow(clippy::too_many_arguments)] -async fn get_serialized_line_chart_data_resolution_dispatch( - db: &DatabaseConnection, - chart_name: String, - resolution: ResolutionKind, - from: Option, - to: Option, - points_limit: Option, - policy: MissingDatePolicy, - mark_approx: u64, -) -> Result, ReadError> { - match resolution { - ResolutionKind::Day => { - get_serialized_line_chart_data::( - db, - chart_name, - from, - to, - points_limit, - policy, - mark_approx, - ) - .await - } - ResolutionKind::Week => { - get_serialized_line_chart_data::( - db, - chart_name, - from, - to, - points_limit, - policy, - mark_approx, - ) - .await - } - ResolutionKind::Month => { - get_serialized_line_chart_data::( - db, - chart_name, - from, - to, - points_limit, - policy, - mark_approx, - ) - .await - } - ResolutionKind::Year => { - get_serialized_line_chart_data::( - db, - chart_name, - from, - to, - points_limit, - policy, - mark_approx, - ) - .await - } - } -} - impl ReadService { async fn query_with_handle( &self, query_handle: QuerySerializedDyn, + range: UniversalRange>, + points_limit: Option, query_time: DateTime, - ) -> anyhow::Result { + ) -> Result { let migrations = BlockscoutMigrations::query_from_db(&self.blockscout) .await - .context("querying migrations from db")?; + .map_err(UpdateError::BlockscoutDB)?; let context = UpdateContext::from_params_now_or_override(UpdateParameters { db: &self.db, blockscout: &self.blockscout, @@ -208,19 +115,20 @@ impl ReadService { force_full: false, }); query_handle - .query_data(&context, None, true) + .query_data(&context, range, points_limit, true) .await - .context("querying data") } async fn query_counter_with_handle( &self, - name: &str, + name: String, settings: EnabledChartSettings, - query_handle: QuerySerializedDyn>, + query_handle: CounterHandle, query_time: DateTime, ) -> anyhow::Result { - let point = self.query_with_handle(query_handle, query_time).await?; + let point = self + .query_with_handle(query_handle, UniversalRange::full(), None, query_time) + .await?; Ok(proto_v1::Counter { id: name.to_string(), value: point.value, @@ -229,6 +137,24 @@ impl ReadService { units: settings.units, }) } + + async fn query_line_chart_with_handle( + &self, + name: String, + chart_entry: &EnabledChartEntry, + query_handle: LineHandle, + range: UniversalRange>, + points_limit: Option, + query_time: DateTime, + ) -> Result { + let data = self + .query_with_handle(query_handle, range, points_limit, query_time) + .await?; + Ok(proto_v1::LineChart { + chart: data, + info: Some(chart_entry.build_proto_line_chart_info(name.to_string())), + }) + } } #[async_trait] @@ -257,7 +183,7 @@ impl StatsService for ReadService { .clone() .into_counter_handle()?; Some(self.query_counter_with_handle( - name, + name.clone(), counter.settings.clone(), query_handle, now.clone(), @@ -295,17 +221,11 @@ impl StatsService for ReadService { Status::not_found(format!("chart with name '{}' was not found", chart_name)) })?; let resolution_info = chart_entry.resolutions.get(&resolution); - let (query_handle, policy, mark_approx) = resolution_info + // settings such as `approximate_trailing_points` and `missing_date_policy` + // are used in the type underneath the handle + let query_handle = resolution_info .and_then(|enabled_resolution| { - let handle = enabled_resolution - .type_specifics - .clone() - .into_line_handle()?; - Some(( - handle, - enabled_resolution.missing_date_policy, - enabled_resolution.approximate_trailing_points, - )) + enabled_resolution.type_specifics.clone().into_line_handle() }) .ok_or_else(|| { Status::not_found(format!( @@ -317,37 +237,28 @@ impl StatsService for ReadService { let from = request .from - .and_then(|date| NaiveDate::from_str(&date).ok()); - let to = request.to.and_then(|date| NaiveDate::from_str(&date).ok()); + .and_then(|date| NaiveDate::from_str(&date).ok()) + .map(|d| day_start(&d)); + let to_exclusive = request + .to + .and_then(|date| NaiveDate::from_str(&date).ok()) + .map(|d| day_start(&d.saturating_next_timespan())); + let request_range = (from..to_exclusive).into(); let points_limit = Some(self.limits.requested_points_limit); - let migrations = BlockscoutMigrations::query_from_db(&self.blockscout) + let chart_data = self + .query_line_chart_with_handle( + chart_name, + chart_entry, + query_handle, + request_range, + points_limit, + Utc::now(), + ) .await - .map_err(|e| map_read_error(ReadError::DB(e)))?; - let context = UpdateContext::from_params_now_or_override(UpdateParameters { - db: &self.db, - blockscout: &self.blockscout, - blockscout_applied_migrations: migrations, - update_time_override: Some(Utc::now()), - force_full: false, - }); - let data = query_handle.query_data().await.map_err(map_read_error)?; - let serialized_chart = get_serialized_line_chart_data_resolution_dispatch( - &self.db, - chart_name.clone(), - resolution, - from, - to, - points_limit, - policy, - mark_approx, - ) - .await - .map_err(map_read_error)?; - Ok(Response::new(proto_v1::LineChart { - chart: serialized_chart, - info: Some(chart_entry.build_proto_line_chart_info(chart_name)), - })) + .map_err(map_update_error)?; + + Ok(Response::new(chart_data)) } async fn get_line_charts( diff --git a/stats/stats/src/charts/chart.rs b/stats/stats/src/charts/chart.rs index 52e2e4457..b3ea22ff1 100644 --- a/stats/stats/src/charts/chart.rs +++ b/stats/stats/src/charts/chart.rs @@ -17,6 +17,7 @@ use super::{ query_dispatch::{ChartTypeSpecifics, QuerySerialized, QuerySerializedDyn}, }; +// todo: rename to `ChartError` or similar #[derive(Error, Debug)] pub enum UpdateError { #[error("blockscout database error: {0}")] diff --git a/stats/stats/src/charts/counters/average_block_time.rs b/stats/stats/src/charts/counters/average_block_time.rs index 0e4fb595f..761b34346 100644 --- a/stats/stats/src/charts/counters/average_block_time.rs +++ b/stats/stats/src/charts/counters/average_block_time.rs @@ -1,4 +1,4 @@ -use std::{cmp::Reverse, ops::Range}; +use std::cmp::Reverse; use crate::{ data_source::{ @@ -9,13 +9,14 @@ use crate::{ }, UpdateContext, }, + range::UniversalRange, types::TimespanValue, utils::NANOS_PER_SEC, ChartProperties, MissingDatePolicy, Named, UpdateError, }; use blockscout_db::entity::blocks; -use chrono::NaiveDate; +use chrono::{DateTime, NaiveDate, Utc}; use entity::sea_orm_active_enums::ChartType; use itertools::Itertools; use sea_orm::{prelude::*, DbBackend, FromQueryResult, QueryOrder, QuerySelect, Statement}; @@ -67,7 +68,7 @@ impl RemoteQueryBehaviour for AverageBlockTimeQuery { async fn query_data( cx: &UpdateContext<'_>, - _range: Option>, + _range: UniversalRange>, ) -> Result, UpdateError> { match query_average_block_time(cx, OFFSET_BLOCKS).await? { Some(avg_block_time) => Ok(avg_block_time), diff --git a/stats/stats/src/charts/counters/mock.rs b/stats/stats/src/charts/counters/mock.rs index 2c138da64..0576f2443 100644 --- a/stats/stats/src/charts/counters/mock.rs +++ b/stats/stats/src/charts/counters/mock.rs @@ -1,4 +1,4 @@ -use std::{marker::PhantomData, ops::Range}; +use std::marker::PhantomData; use crate::{ data_source::{ @@ -9,13 +9,13 @@ use crate::{ types::Get, UpdateContext, }, + range::UniversalRange, types::timespans::DateValue, ChartProperties, Named, UpdateError, }; use chrono::{DateTime, NaiveDate, Utc}; use entity::sea_orm_active_enums::ChartType; -use sea_orm::prelude::DateTimeUtc; pub struct MockCounterRetrieve(PhantomData<(PointDateTime, Value)>) where @@ -31,7 +31,7 @@ where async fn query_data( cx: &UpdateContext<'_>, - _range: Option>, + _range: UniversalRange>, ) -> Result { if cx.time >= PointDateTime::get() { Ok(DateValue:: { diff --git a/stats/stats/src/charts/counters/total_blocks.rs b/stats/stats/src/charts/counters/total_blocks.rs index bc9707c7d..b1a0b37dc 100644 --- a/stats/stats/src/charts/counters/total_blocks.rs +++ b/stats/stats/src/charts/counters/total_blocks.rs @@ -1,4 +1,3 @@ -use std::ops::Range; use crate::{ charts::db_interaction::read::get_estimated_table_rows, @@ -9,12 +8,13 @@ use crate::{ }, types::UpdateContext, }, + range::UniversalRange, types::timespans::DateValue, ChartProperties, MissingDatePolicy, Named, UpdateError, }; use blockscout_db::entity::blocks; -use chrono::{NaiveDate, NaiveDateTime, Utc}; +use chrono::{DateTime, NaiveDate, NaiveDateTime, Utc}; use entity::sea_orm_active_enums::ChartType; use sea_orm::{prelude::*, sea_query::Expr, FromQueryResult, QuerySelect}; @@ -31,7 +31,7 @@ impl RemoteQueryBehaviour for TotalBlocksQueryBehaviour { async fn query_data( cx: &UpdateContext<'_>, - _range: Option>, + _range: UniversalRange>, ) -> Result { let data = blocks::Entity::find() .select_only() diff --git a/stats/stats/src/charts/counters/total_contracts.rs b/stats/stats/src/charts/counters/total_contracts.rs index 1bba5fb43..a28541205 100644 --- a/stats/stats/src/charts/counters/total_contracts.rs +++ b/stats/stats/src/charts/counters/total_contracts.rs @@ -1,4 +1,3 @@ -use std::ops::Range; use crate::{ data_source::{ @@ -8,12 +7,13 @@ use crate::{ }, UpdateContext, }, + range::UniversalRange, types::timespans::DateValue, ChartProperties, MissingDatePolicy, Named, UpdateError, }; use blockscout_db::entity::addresses; -use chrono::NaiveDate; +use chrono::{DateTime, NaiveDate, Utc}; use entity::sea_orm_active_enums::ChartType; use sea_orm::prelude::*; @@ -24,7 +24,7 @@ impl RemoteQueryBehaviour for TotalContractsQueryBehaviour { async fn query_data( cx: &UpdateContext<'_>, - _range: Option>, + _range: UniversalRange>, ) -> Result { let value = addresses::Entity::find() .filter(addresses::Column::ContractCode.is_not_null()) diff --git a/stats/stats/src/charts/counters/yesterday_txns.rs b/stats/stats/src/charts/counters/yesterday_txns.rs index 6ef5131dc..929c4f72f 100644 --- a/stats/stats/src/charts/counters/yesterday_txns.rs +++ b/stats/stats/src/charts/counters/yesterday_txns.rs @@ -1,4 +1,3 @@ -use std::ops::Range; use crate::{ data_source::{ @@ -9,6 +8,7 @@ use crate::{ UpdateContext, }, lines::NewTxnsStatement, + range::UniversalRange, types::TimespanValue, utils::day_start, ChartProperties, MissingDatePolicy, Named, UpdateError, @@ -24,7 +24,7 @@ impl RemoteQueryBehaviour for YesterdayTxnsQuery { async fn query_data( cx: &UpdateContext<'_>, - _range: Option>>, + _range: UniversalRange>, ) -> Result { let today = cx.time.date_naive(); let yesterday = today diff --git a/stats/stats/src/charts/db_interaction/read.rs b/stats/stats/src/charts/db_interaction/read.rs index 2bf4d5166..8e09f3c43 100644 --- a/stats/stats/src/charts/db_interaction/read.rs +++ b/stats/stats/src/charts/db_interaction/read.rs @@ -5,11 +5,11 @@ use crate::{ UpdateContext, }, missing_date::{fill_and_filter_chart, fit_into_range}, + range::{exclusive_range_to_inclusive, UniversalRange}, types::{ timespans::{DateValue, Month, Week, Year}, ExtendedTimespanValue, Timespan, TimespanDuration, TimespanValue, }, - utils::exclusive_datetime_range_to_inclusive, ChartProperties, MissingDatePolicy, UpdateError, }; @@ -196,7 +196,7 @@ fn relevant_data_until( R::from_date(t.date_naive()).saturating_start_timestamp() == t; // last_updated_at timestamp is not included in the range let inclusive_last_updated_at_end = - exclusive_datetime_range_to_inclusive(DateTime::::MIN_UTC..t); + exclusive_range_to_inclusive(DateTime::::MIN_UTC..t); ( Some(R::from_date( inclusive_last_updated_at_end.end().date_naive(), @@ -697,7 +697,7 @@ impl RemoteQueryBehaviour for QueryAllBlockTimestampRange { async fn query_data( cx: &UpdateContext<'_>, - _range: Option>>, + _range: UniversalRange>, ) -> Result { let start_timestamp = get_min_date_blockscout(cx.blockscout) .await diff --git a/stats/stats/src/charts/lines/active_accounts.rs b/stats/stats/src/charts/lines/active_accounts.rs index 51162792f..19a91b285 100644 --- a/stats/stats/src/charts/lines/active_accounts.rs +++ b/stats/stats/src/charts/lines/active_accounts.rs @@ -3,6 +3,7 @@ use std::ops::Range; use crate::{ + charts::db_interaction::read::QueryAllBlockTimestampRange, data_source::{ kinds::{ local_db::{ @@ -66,8 +67,9 @@ impl StatementFromRange for ActiveAccountsStatement { } } -pub type ActiveAccountsRemote = - RemoteDatabaseSource>; +pub type ActiveAccountsRemote = RemoteDatabaseSource< + PullAllWithAndSort, +>; pub struct Properties; diff --git a/stats/stats/src/charts/lines/average_block_rewards.rs b/stats/stats/src/charts/lines/average_block_rewards.rs index 439cebfd4..ff55b9b27 100644 --- a/stats/stats/src/charts/lines/average_block_rewards.rs +++ b/stats/stats/src/charts/lines/average_block_rewards.rs @@ -1,6 +1,7 @@ use std::ops::Range; use crate::{ + charts::db_interaction::read::QueryAllBlockTimestampRange, data_source::{ kinds::{ data_manipulation::{ @@ -55,8 +56,9 @@ impl StatementFromRange for AverageBlockRewardsQuery { } } -pub type AverageBlockRewardsRemote = - RemoteDatabaseSource>; +pub type AverageBlockRewardsRemote = RemoteDatabaseSource< + PullAllWithAndSort, +>; pub type AverageBlockRewardsRemoteString = MapToString; diff --git a/stats/stats/src/charts/lines/average_block_size.rs b/stats/stats/src/charts/lines/average_block_size.rs index 848703271..b637142d4 100644 --- a/stats/stats/src/charts/lines/average_block_size.rs +++ b/stats/stats/src/charts/lines/average_block_size.rs @@ -1,6 +1,7 @@ use std::ops::Range; use crate::{ + charts::db_interaction::read::QueryAllBlockTimestampRange, data_source::{ kinds::{ data_manipulation::{ @@ -52,8 +53,9 @@ impl StatementFromRange for AverageBlockSizeStatement { } } -pub type AverageBlockSizeRemote = - RemoteDatabaseSource>; +pub type AverageBlockSizeRemote = RemoteDatabaseSource< + PullAllWithAndSort, +>; pub struct Properties; diff --git a/stats/stats/src/charts/lines/average_gas_limit.rs b/stats/stats/src/charts/lines/average_gas_limit.rs index d9d762675..a397123fb 100644 --- a/stats/stats/src/charts/lines/average_gas_limit.rs +++ b/stats/stats/src/charts/lines/average_gas_limit.rs @@ -1,6 +1,7 @@ use std::ops::Range; use crate::{ + charts::db_interaction::read::QueryAllBlockTimestampRange, data_source::{ kinds::{ data_manipulation::{ @@ -52,8 +53,9 @@ impl StatementFromRange for AverageGasLimitStatement { } } -pub type AverageGasLimitRemote = - RemoteDatabaseSource>; +pub type AverageGasLimitRemote = RemoteDatabaseSource< + PullAllWithAndSort, +>; pub struct Properties; diff --git a/stats/stats/src/charts/lines/average_gas_price.rs b/stats/stats/src/charts/lines/average_gas_price.rs index 0ab1afec5..5b5f2b6e7 100644 --- a/stats/stats/src/charts/lines/average_gas_price.rs +++ b/stats/stats/src/charts/lines/average_gas_price.rs @@ -1,6 +1,7 @@ use std::ops::Range; use crate::{ + charts::db_interaction::read::QueryAllBlockTimestampRange, data_source::{ kinds::{ data_manipulation::{ @@ -105,8 +106,9 @@ impl StatementFromRange for AverageGasPriceStatement { } } -pub type AverageGasPriceRemote = - RemoteDatabaseSource>; +pub type AverageGasPriceRemote = RemoteDatabaseSource< + PullAllWithAndSort, +>; pub type AverageGasPriceRemoteString = MapToString; diff --git a/stats/stats/src/charts/lines/average_txn_fee.rs b/stats/stats/src/charts/lines/average_txn_fee.rs index 4d3cb09f1..6b678cfe2 100644 --- a/stats/stats/src/charts/lines/average_txn_fee.rs +++ b/stats/stats/src/charts/lines/average_txn_fee.rs @@ -3,6 +3,7 @@ use std::ops::Range; use crate::{ + charts::db_interaction::read::QueryAllBlockTimestampRange, data_source::{ kinds::{ data_manipulation::{ @@ -108,8 +109,9 @@ impl StatementFromRange for AverageTxnFeeStatement { } } -pub type AverageTxnFeeRemote = - RemoteDatabaseSource>; +pub type AverageTxnFeeRemote = RemoteDatabaseSource< + PullAllWithAndSort, +>; pub type AverageTxnFeeRemoteString = MapToString; diff --git a/stats/stats/src/charts/lines/gas_used_growth.rs b/stats/stats/src/charts/lines/gas_used_growth.rs index 3ff92ba6b..208f64d66 100644 --- a/stats/stats/src/charts/lines/gas_used_growth.rs +++ b/stats/stats/src/charts/lines/gas_used_growth.rs @@ -1,6 +1,7 @@ use std::ops::Range; use crate::{ + charts::db_interaction::read::QueryAllBlockTimestampRange, data_source::{ kinds::{ data_manipulation::{ @@ -51,8 +52,9 @@ impl StatementFromRange for GasUsedPartialStatement { } } -pub type GasUsedPartialRemote = - RemoteDatabaseSource>; +pub type GasUsedPartialRemote = RemoteDatabaseSource< + PullAllWithAndSort, +>; pub struct IncrementsFromPartialSum; diff --git a/stats/stats/src/charts/lines/mock.rs b/stats/stats/src/charts/lines/mock.rs index f4f7a611b..2dcecb1c3 100644 --- a/stats/stats/src/charts/lines/mock.rs +++ b/stats/stats/src/charts/lines/mock.rs @@ -10,6 +10,7 @@ use crate::{ UpdateContext, }, missing_date::fit_into_range, + range::{Incrementable, UniversalRange}, types::{timespans::DateValue, Timespan, TimespanValue}, ChartProperties, MissingDatePolicy, Named, UpdateError, }; @@ -17,8 +18,10 @@ use crate::{ use chrono::{DateTime, Duration, NaiveDate, Utc}; use entity::sea_orm_active_enums::ChartType; use rand::{distributions::uniform::SampleUniform, rngs::StdRng, Rng, SeedableRng}; -use sea_orm::prelude::*; -use std::{marker::PhantomData, ops::Range}; +use std::{ + marker::PhantomData, + ops::{Bound, Range}, +}; /// non-inclusive range fn generate_intervals(mut start: NaiveDate, end: NaiveDate) -> Vec { @@ -51,7 +54,7 @@ pub fn mocked_lines( pub fn mock_trim_lines( data: Vec>, query_time: DateTime, - query_range: Option>, + query_range: UniversalRange>, policy: MissingDatePolicy, ) -> Vec> where @@ -59,11 +62,16 @@ where V: Clone, { let date_range_start = query_range - .clone() - .map(|r| T::from_date(r.start.date_naive())); + .start + .map(|start| T::from_date(start.date_naive())); let mut date_range_end = query_time; - if let Some(r) = query_range { - date_range_end = date_range_end.min(r.end) + let query_range_end_exclusive = match query_range.end { + Bound::Included(end) => Some(end.saturating_inc()), + Bound::Excluded(end) => Some(end), + Bound::Unbounded => None, + }; + if let Some(end_exclusive) = query_range_end_exclusive { + date_range_end = date_range_end.min(end_exclusive) } fit_into_range( data, @@ -91,7 +99,7 @@ where async fn query_data( cx: &UpdateContext<'_>, - range: Option>, + range: UniversalRange>, ) -> Result>, UpdateError> { let full_data = mocked_lines(DateRange::get(), ValueRange::get()); Ok(mock_trim_lines(full_data, cx.time, range, Policy::get())) @@ -137,7 +145,7 @@ where async fn query_data( cx: &UpdateContext<'_>, - range: Option>, + range: UniversalRange>, ) -> Result { Ok(mock_trim_lines(Data::get(), cx.time, range, Policy::get())) } diff --git a/stats/stats/src/charts/lines/native_coin_supply.rs b/stats/stats/src/charts/lines/native_coin_supply.rs index 6fccfb654..5b537c59d 100644 --- a/stats/stats/src/charts/lines/native_coin_supply.rs +++ b/stats/stats/src/charts/lines/native_coin_supply.rs @@ -1,6 +1,7 @@ use std::ops::Range; use crate::{ + charts::db_interaction::read::QueryAllBlockTimestampRange, data_source::{ kinds::{ data_manipulation::{ @@ -89,8 +90,9 @@ impl StatementFromRange for NativeCoinSupplyStatement { } // query returns float value -pub type NativeCoinSupplyRemote = - RemoteDatabaseSource>; +pub type NativeCoinSupplyRemote = RemoteDatabaseSource< + PullAllWithAndSort, +>; pub type NativeCoinSupplyRemoteString = MapToString; diff --git a/stats/stats/src/charts/lines/new_accounts.rs b/stats/stats/src/charts/lines/new_accounts.rs index 553d0e410..2266e0a04 100644 --- a/stats/stats/src/charts/lines/new_accounts.rs +++ b/stats/stats/src/charts/lines/new_accounts.rs @@ -1,7 +1,7 @@ use std::ops::Range; use crate::{ - charts::types::timespans::DateValue, + charts::{db_interaction::read::QueryAllBlockTimestampRange, types::timespans::DateValue}, data_source::{ kinds::{ data_manipulation::{ @@ -21,12 +21,13 @@ use crate::{ }, define_and_impl_resolution_properties, missing_date::trim_out_of_range_sorted, + range::{data_source_query_range_to_db_statement_range, UniversalRange}, types::timespans::{Month, Week, Year}, utils::sql_with_range_filter_opt, ChartProperties, Named, UpdateError, }; -use chrono::NaiveDate; +use chrono::{DateTime, NaiveDate, Utc}; use entity::sea_orm_active_enums::ChartType; use sea_orm::{prelude::*, DbBackend, FromQueryResult, Statement}; @@ -99,17 +100,22 @@ impl RemoteQueryBehaviour for NewAccountsQueryBehaviour { async fn query_data( cx: &UpdateContext<'_>, - range: Option>, + range: UniversalRange>, ) -> Result>, UpdateError> { - let query = - NewAccountsStatement::get_statement(range.clone(), &cx.blockscout_applied_migrations); + let statement_range = + data_source_query_range_to_db_statement_range::(cx, range) + .await?; + let query = NewAccountsStatement::get_statement( + statement_range.clone(), + &cx.blockscout_applied_migrations, + ); let mut data = DateValue::::find_by_statement(query) .all(cx.blockscout) .await .map_err(UpdateError::BlockscoutDB)?; // make sure that it's sorted data.sort_by_key(|d| d.timespan); - if let Some(range) = range { + if let Some(range) = statement_range { let range = range.start.date_naive()..=range.end.date_naive(); trim_out_of_range_sorted(&mut data, range); } diff --git a/stats/stats/src/charts/lines/new_block_rewards.rs b/stats/stats/src/charts/lines/new_block_rewards.rs index cc0e469c5..402537b37 100644 --- a/stats/stats/src/charts/lines/new_block_rewards.rs +++ b/stats/stats/src/charts/lines/new_block_rewards.rs @@ -6,6 +6,7 @@ use std::ops::Range; use crate::{ + charts::db_interaction::read::QueryAllBlockTimestampRange, data_source::{ kinds::{ data_manipulation::{ @@ -53,8 +54,9 @@ impl StatementFromRange for NewBlockRewardsStatement { } } -pub type NewBlockRewardsRemote = - RemoteDatabaseSource>; +pub type NewBlockRewardsRemote = RemoteDatabaseSource< + PullAllWithAndSort, +>; pub struct Properties; @@ -96,6 +98,7 @@ mod tests { use super::*; use crate::{ data_source::{types::BlockscoutMigrations, DataSource, UpdateContext, UpdateParameters}, + range::UniversalRange, tests::{ init_db::init_db_all, mock_blockscout::fill_mock_blockscout_data, @@ -154,12 +157,13 @@ mod tests { .await .unwrap(); let mut timer = AggregateTimer::new(); - let data: Vec<_> = NewBlockRewardsMonthlyInt::query_data(&cx, None, &mut timer) - .await - .unwrap() - .into_iter() - .map(|p| (p.timespan.into_date().to_string(), p.value.to_string())) - .collect(); + let data: Vec<_> = + NewBlockRewardsMonthlyInt::query_data(&cx, UniversalRange::full(), &mut timer) + .await + .unwrap() + .into_iter() + .map(|p| (p.timespan.into_date().to_string(), p.value.to_string())) + .collect(); assert_eq!(data, expected); } } diff --git a/stats/stats/src/charts/lines/new_blocks.rs b/stats/stats/src/charts/lines/new_blocks.rs index 1e35fbe08..414e70d22 100644 --- a/stats/stats/src/charts/lines/new_blocks.rs +++ b/stats/stats/src/charts/lines/new_blocks.rs @@ -1,6 +1,7 @@ use std::ops::Range; use crate::{ + charts::db_interaction::read::QueryAllBlockTimestampRange, data_source::{ kinds::{ data_manipulation::{ @@ -50,8 +51,9 @@ impl StatementFromRange for NewBlocksStatement { } } -pub type NewBlocksRemote = - RemoteDatabaseSource>; +pub type NewBlocksRemote = RemoteDatabaseSource< + PullAllWithAndSort, +>; pub struct Properties; diff --git a/stats/stats/src/charts/lines/new_contracts.rs b/stats/stats/src/charts/lines/new_contracts.rs index 00226bb50..958d3893f 100644 --- a/stats/stats/src/charts/lines/new_contracts.rs +++ b/stats/stats/src/charts/lines/new_contracts.rs @@ -1,6 +1,7 @@ use std::ops::Range; use crate::{ + charts::db_interaction::read::QueryAllBlockTimestampRange, data_source::{ kinds::{ data_manipulation::{ @@ -116,8 +117,9 @@ impl StatementFromRange for NewContractsStatement { } } -pub type NewContractsRemote = - RemoteDatabaseSource>; +pub type NewContractsRemote = RemoteDatabaseSource< + PullAllWithAndSort, +>; pub struct Properties; diff --git a/stats/stats/src/charts/lines/new_native_coin_transfers.rs b/stats/stats/src/charts/lines/new_native_coin_transfers.rs index 7f89afff1..2120256a5 100644 --- a/stats/stats/src/charts/lines/new_native_coin_transfers.rs +++ b/stats/stats/src/charts/lines/new_native_coin_transfers.rs @@ -1,6 +1,7 @@ use std::ops::Range; use crate::{ + charts::db_interaction::read::QueryAllBlockTimestampRange, data_source::{ kinds::{ data_manipulation::{ @@ -77,8 +78,14 @@ impl StatementFromRange for NewNativeCoinTransfersStatement { } } -pub type NewNativeCoinTransfersRemote = - RemoteDatabaseSource>; +pub type NewNativeCoinTransfersRemote = RemoteDatabaseSource< + PullAllWithAndSort< + NewNativeCoinTransfersStatement, + NaiveDate, + String, + QueryAllBlockTimestampRange, + >, +>; pub struct Properties; diff --git a/stats/stats/src/charts/lines/new_txns.rs b/stats/stats/src/charts/lines/new_txns.rs index 6dec3c470..e4ef6a5c5 100644 --- a/stats/stats/src/charts/lines/new_txns.rs +++ b/stats/stats/src/charts/lines/new_txns.rs @@ -1,6 +1,7 @@ use std::ops::Range; use crate::{ + charts::db_interaction::read::QueryAllBlockTimestampRange, data_source::{ kinds::{ data_manipulation::{ @@ -73,8 +74,9 @@ impl StatementFromRange for NewTxnsStatement { } } -pub type NewTxnsRemote = - RemoteDatabaseSource>; +pub type NewTxnsRemote = RemoteDatabaseSource< + PullAllWithAndSort, +>; pub struct Properties; diff --git a/stats/stats/src/charts/lines/new_verified_contracts.rs b/stats/stats/src/charts/lines/new_verified_contracts.rs index 211b8644f..2778587a8 100644 --- a/stats/stats/src/charts/lines/new_verified_contracts.rs +++ b/stats/stats/src/charts/lines/new_verified_contracts.rs @@ -1,6 +1,7 @@ use std::ops::Range; use crate::{ + charts::db_interaction::read::QueryAllBlockTimestampRange, data_source::{ kinds::{ data_manipulation::{ @@ -48,8 +49,14 @@ impl StatementFromRange for NewVerifiedContractsStatement { } } -pub type NewVerifiedContractsRemote = - RemoteDatabaseSource>; +pub type NewVerifiedContractsRemote = RemoteDatabaseSource< + PullAllWithAndSort< + NewVerifiedContractsStatement, + NaiveDate, + String, + QueryAllBlockTimestampRange, + >, +>; pub struct Properties; diff --git a/stats/stats/src/charts/lines/txns_fee.rs b/stats/stats/src/charts/lines/txns_fee.rs index aabe549bc..ad4326591 100644 --- a/stats/stats/src/charts/lines/txns_fee.rs +++ b/stats/stats/src/charts/lines/txns_fee.rs @@ -3,6 +3,7 @@ use std::ops::Range; use crate::{ + charts::db_interaction::read::QueryAllBlockTimestampRange, data_source::{ kinds::{ data_manipulation::{ @@ -106,7 +107,9 @@ impl StatementFromRange for TxnsFeeStatement { } } -pub type TxnsFeeRemote = RemoteDatabaseSource>; +pub type TxnsFeeRemote = RemoteDatabaseSource< + PullAllWithAndSort, +>; pub type TxnsFeeRemoteString = MapToString; diff --git a/stats/stats/src/charts/lines/txns_success_rate.rs b/stats/stats/src/charts/lines/txns_success_rate.rs index 71ab391d6..1df1d3370 100644 --- a/stats/stats/src/charts/lines/txns_success_rate.rs +++ b/stats/stats/src/charts/lines/txns_success_rate.rs @@ -1,6 +1,7 @@ use std::ops::Range; use crate::{ + charts::db_interaction::read::QueryAllBlockTimestampRange, data_source::{ kinds::{ data_manipulation::{ @@ -81,8 +82,9 @@ impl StatementFromRange for TxnsSuccessRateStatement { } } -pub type TxnsSuccessRateRemote = - RemoteDatabaseSource>; +pub type TxnsSuccessRateRemote = RemoteDatabaseSource< + PullAllWithAndSort, +>; pub type TxnsSuccessRateRemoteString = MapToString; diff --git a/stats/stats/src/charts/query_dispatch.rs b/stats/stats/src/charts/query_dispatch.rs index e90b8ad8b..8b79f409a 100644 --- a/stats/stats/src/charts/query_dispatch.rs +++ b/stats/stats/src/charts/query_dispatch.rs @@ -1,4 +1,4 @@ -use std::{fmt::Debug, future::Future, ops::Range, pin::Pin, sync::Arc}; +use std::{fmt::Debug, future::Future, pin::Pin, sync::Arc}; use chrono::{DateTime, NaiveDate, Utc}; use entity::sea_orm_active_enums::ChartType; @@ -12,7 +12,8 @@ use crate::{ }, DataSource, UpdateContext, }, - exclusive_datetime_range_to_inclusive, RequestedPointsLimit, + range::{exclusive_range_to_inclusive, UniversalRange}, + RequestedPointsLimit, }; use super::{ @@ -29,8 +30,7 @@ pub trait QuerySerialized { fn query_data<'a>( &self, cx: &UpdateContext<'a>, - from: Option>, - to: Option>, + range: UniversalRange>, points_limit: Option, fill_missing_dates: bool, ) -> Pin> + Send + 'a>>; @@ -132,8 +132,7 @@ where fn query_data<'a>( &self, cx: &UpdateContext<'a>, - from: Option>, - to: Option>, + range: UniversalRange>, points_limit: Option, fill_missing_dates: bool, ) -> Pin> + Send + 'a>> @@ -149,7 +148,7 @@ where fn serialize_point( point: ExtendedTimespanValue, ) -> Point { - let time_range = exclusive_datetime_range_to_inclusive(point.timespan.into_time_range()); + let time_range = exclusive_range_to_inclusive(point.timespan.into_time_range()); let date_range = { time_range.start().date_naive()..=time_range.end().date_naive() }; Point { date: date_range.start().to_string(), diff --git a/stats/stats/src/data_source/kinds/auxiliary/cumulative.rs b/stats/stats/src/data_source/kinds/auxiliary/cumulative.rs index dc21d253d..3b13235eb 100644 --- a/stats/stats/src/data_source/kinds/auxiliary/cumulative.rs +++ b/stats/stats/src/data_source/kinds/auxiliary/cumulative.rs @@ -1,6 +1,6 @@ use std::{ marker::PhantomData, - ops::{AddAssign, Range}, + ops::AddAssign, }; use blockscout_metrics_tools::AggregateTimer; @@ -11,6 +11,7 @@ use sea_orm::DatabaseConnection; use crate::{ data_processing::cumsum, data_source::{DataSource, UpdateContext}, + range::UniversalRange, types::TimespanValue, UpdateError, }; @@ -52,7 +53,7 @@ where async fn query_data( cx: &UpdateContext<'_>, - range: Option>>, + range: UniversalRange>, dependency_data_fetch_timer: &mut AggregateTimer, ) -> Result { let delta_data = Delta::query_data(cx, range, dependency_data_fetch_timer).await?; diff --git a/stats/stats/src/data_source/kinds/data_manipulation/delta.rs b/stats/stats/src/data_source/kinds/data_manipulation/delta.rs index 9b6edb65d..b741289a8 100644 --- a/stats/stats/src/data_source/kinds/data_manipulation/delta.rs +++ b/stats/stats/src/data_source/kinds/data_manipulation/delta.rs @@ -6,18 +6,19 @@ use std::{ fmt::Display, marker::PhantomData, - ops::{Range, SubAssign}, + ops::SubAssign, str::FromStr, }; use blockscout_metrics_tools::AggregateTimer; use chrono::{DateTime, TimeDelta, Utc}; use rust_decimal::prelude::Zero; -use sea_orm::{prelude::DateTimeUtc, DatabaseConnection, DbErr}; +use sea_orm::{DatabaseConnection, DbErr}; use crate::{ data_processing::deltas, data_source::{DataSource, UpdateContext}, + range::UniversalRange, types::TimespanValue, UpdateError, }; @@ -63,19 +64,18 @@ where async fn query_data( cx: &UpdateContext<'_>, - range: Option>, + range: UniversalRange>, dependency_data_fetch_timer: &mut AggregateTimer, ) -> Result { - let request_range = range.clone().map(|r| { - let start = r - .start - .checked_sub_signed(TimeDelta::days(1)) - .unwrap_or(DateTime::::MAX_UTC); - let end = r.end; - start..end + let mut request_range = range.clone(); + request_range.start = request_range.start.map(|s| { + s.checked_sub_signed(TimeDelta::days(1)) + .unwrap_or(DateTime::::MAX_UTC) }); + let start_is_bounded = request_range.start.is_some(); + let cum_data = DS::query_data(cx, request_range, dependency_data_fetch_timer).await?; - let (prev_value, cum_data) = if range.is_some() { + let (prev_value, cum_data) = if start_is_bounded { let mut cum_data = cum_data.into_iter(); let Some(range_start) = cum_data.next() else { tracing::warn!("Value before the range was not found, finishing update"); diff --git a/stats/stats/src/data_source/kinds/data_manipulation/filter_deducible.rs b/stats/stats/src/data_source/kinds/data_manipulation/filter_deducible.rs index 7b4b5786d..645657c30 100644 --- a/stats/stats/src/data_source/kinds/data_manipulation/filter_deducible.rs +++ b/stats/stats/src/data_source/kinds/data_manipulation/filter_deducible.rs @@ -1,7 +1,7 @@ //! Filter points that can be deduced according to `MissingDatePolicy`. //! Can help with space usage efficiency. -use std::{marker::PhantomData, ops::Range}; +use std::marker::PhantomData; use blockscout_metrics_tools::AggregateTimer; use chrono::{DateTime, Utc}; @@ -10,6 +10,7 @@ use sea_orm::DatabaseConnection; use crate::{ data_source::{DataSource, UpdateContext}, + range::UniversalRange, types::TimespanValue, ChartProperties, MissingDatePolicy, UpdateError, }; @@ -51,7 +52,7 @@ where async fn query_data( cx: &UpdateContext<'_>, - range: Option>>, + range: UniversalRange>, dependency_data_fetch_timer: &mut AggregateTimer, ) -> Result { let data = DS::query_data(cx, range, dependency_data_fetch_timer).await?; @@ -83,6 +84,7 @@ mod tests { data_source::types::BlockscoutMigrations, gettable_const, lines::PredefinedMockSource, + range::UniversalRange, tests::point_construction::{d_v_double, dt}, types::timespans::DateValue, MissingDatePolicy, Named, @@ -161,9 +163,13 @@ mod tests { force_full: false, }; assert_eq!( - ::query_data(&context, None, &mut AggregateTimer::new()) - .await - .unwrap(), + ::query_data( + &context, + UniversalRange::full(), + &mut AggregateTimer::new() + ) + .await + .unwrap(), vec![ d_v_double("2024-07-08", 5.0), d_v_double("2024-07-10", 5.0), @@ -175,9 +181,13 @@ mod tests { ] ); assert_eq!( - ::query_data(&context, None, &mut AggregateTimer::new()) - .await - .unwrap(), + ::query_data( + &context, + UniversalRange::full(), + &mut AggregateTimer::new() + ) + .await + .unwrap(), vec![ d_v_double("2024-07-08", 5.0), d_v_double("2024-07-14", 10.3), diff --git a/stats/stats/src/data_source/kinds/data_manipulation/last_point.rs b/stats/stats/src/data_source/kinds/data_manipulation/last_point.rs index 036651aad..e502f83d2 100644 --- a/stats/stats/src/data_source/kinds/data_manipulation/last_point.rs +++ b/stats/stats/src/data_source/kinds/data_manipulation/last_point.rs @@ -2,14 +2,15 @@ //! //! Takes last data point from some other (vector) source -use std::{marker::PhantomData, ops::Range}; +use std::marker::PhantomData; use blockscout_metrics_tools::AggregateTimer; use chrono::{DateTime, Utc}; -use sea_orm::{prelude::DateTimeUtc, DatabaseConnection, DbErr}; +use sea_orm::{DatabaseConnection, DbErr}; use crate::{ data_source::{source::DataSource, UpdateContext}, + range::UniversalRange, types::{Timespan, TimespanValue, ZeroTimespanValue}, utils::day_start, UpdateError, @@ -48,12 +49,12 @@ where async fn query_data( cx: &UpdateContext<'_>, - _range: Option>, + _range: UniversalRange>, dependency_data_fetch_timer: &mut AggregateTimer, ) -> Result { let data = DS::query_data( cx, - Some(day_start(&cx.time.date_naive())..cx.time), + (day_start(&cx.time.date_naive())..cx.time).into(), dependency_data_fetch_timer, ) .await?; diff --git a/stats/stats/src/data_source/kinds/data_manipulation/map/mod.rs b/stats/stats/src/data_source/kinds/data_manipulation/map/mod.rs index 648a86a6f..a4449250d 100644 --- a/stats/stats/src/data_source/kinds/data_manipulation/map/mod.rs +++ b/stats/stats/src/data_source/kinds/data_manipulation/map/mod.rs @@ -1,14 +1,15 @@ //! `map` for a data source, i.e. applies a function to the output //! of some other source. -use std::{marker::PhantomData, ops::Range}; +use std::marker::PhantomData; use blockscout_metrics_tools::AggregateTimer; -use chrono::Utc; -use sea_orm::{prelude::DateTimeUtc, DatabaseConnection, DbErr}; +use chrono::{DateTime, Utc}; +use sea_orm::{DatabaseConnection, DbErr}; use crate::{ data_source::{DataSource, UpdateContext}, + range::UniversalRange, UpdateError, }; @@ -58,7 +59,7 @@ where async fn query_data( cx: &UpdateContext<'_>, - range: Option>, + range: UniversalRange>, dependency_data_fetch_timer: &mut AggregateTimer, ) -> Result { let inner_data = diff --git a/stats/stats/src/data_source/kinds/data_manipulation/resolutions/average.rs b/stats/stats/src/data_source/kinds/data_manipulation/resolutions/average.rs index 7bc8dc642..26c4d1dc4 100644 --- a/stats/stats/src/data_source/kinds/data_manipulation/resolutions/average.rs +++ b/stats/stats/src/data_source/kinds/data_manipulation/resolutions/average.rs @@ -1,15 +1,16 @@ //! Constructors for lower resolutions of average value charts -use std::{cmp::Ordering, fmt::Debug, marker::PhantomData, ops::Range}; +use std::{cmp::Ordering, fmt::Debug, marker::PhantomData}; use blockscout_metrics_tools::AggregateTimer; use chrono::{DateTime, Utc}; use itertools::{EitherOrBoth, Itertools}; -use sea_orm::{prelude::DateTimeUtc, DatabaseConnection, DbErr}; +use sea_orm::{DatabaseConnection, DbErr}; use crate::{ data_source::{ kinds::data_manipulation::resolutions::reduce_each_timespan, DataSource, UpdateContext, }, + range::UniversalRange, types::{ConsistsOf, Timespan, TimespanValue}, UpdateError, }; @@ -34,7 +35,7 @@ impl DataSource where Average: DataSource>>, Weight: DataSource>>, - LowerRes: Timespan + ConsistsOf + Eq + Debug + Send, + LowerRes: Timespan + ConsistsOf + Eq + Ord + Debug + Send, HigherRes: Ord + Clone + Debug + Send, { type MainDependencies = Average; @@ -61,10 +62,10 @@ where async fn query_data( cx: &UpdateContext<'_>, - range: Option>, + range: UniversalRange>, dependency_data_fetch_timer: &mut AggregateTimer, ) -> Result { - let time_range_for_lower_res = range.map(extend_to_timespan_boundaries::); + let time_range_for_lower_res = extend_to_timespan_boundaries::(range); let high_res_averages = Average::query_data( cx, time_range_for_lower_res.clone(), @@ -188,6 +189,8 @@ where #[cfg(test)] mod tests { + use std::ops::Range; + use crate::{ data_source::{kinds::data_manipulation::map::MapParseTo, types::BlockscoutMigrations}, gettable_const, @@ -325,7 +328,7 @@ mod tests { time: dt("2024-07-15T09:00:00").and_utc(), force_full: false, }, - Some(dt("2024-07-08T09:00:00").and_utc()..dt("2024-07-15T00:00:01").and_utc()), + (dt("2024-07-08T09:00:00").and_utc()..dt("2024-07-15T00:00:01").and_utc()).into(), &mut AggregateTimer::new(), ) .await @@ -375,9 +378,13 @@ mod tests { }; let week_1_average = (5.0 * 100.0 + 34.2 * 2.0 + 10.3 * 12.0) / (100.0 + 2.0 + 12.0); assert_eq!( - TestedAverageSource::query_data(&context, None, &mut AggregateTimer::new()) - .await - .unwrap(), + TestedAverageSource::query_data( + &context, + UniversalRange::full(), + &mut AggregateTimer::new() + ) + .await + .unwrap(), vec![ w_v_double("2024-07-08", week_1_average), w_v_double("2024-07-15", 5.0) @@ -420,9 +427,13 @@ mod tests { force_full: false, }; assert_eq!( - TestedAverageSource::query_data(&context, None, &mut AggregateTimer::new()) - .await - .unwrap(), + TestedAverageSource::query_data( + &context, + UniversalRange::full(), + &mut AggregateTimer::new() + ) + .await + .unwrap(), vec![w_v_double("2022-11-07", 0.8888888888888888),] ); } @@ -462,9 +473,13 @@ mod tests { force_full: false, }; assert_eq!( - TestedAverageSource::query_data(&context, None, &mut AggregateTimer::new()) - .await - .unwrap(), + TestedAverageSource::query_data( + &context, + UniversalRange::full(), + &mut AggregateTimer::new() + ) + .await + .unwrap(), vec![w_v_double("2022-11-07", 1.0),] ); } diff --git a/stats/stats/src/data_source/kinds/data_manipulation/resolutions/last_value.rs b/stats/stats/src/data_source/kinds/data_manipulation/resolutions/last_value.rs index 6af0b73a7..35daaa90d 100644 --- a/stats/stats/src/data_source/kinds/data_manipulation/resolutions/last_value.rs +++ b/stats/stats/src/data_source/kinds/data_manipulation/resolutions/last_value.rs @@ -3,14 +3,15 @@ //! Intended for "growth" charts where cumulative number of something //! is presented. -use std::{fmt::Debug, marker::PhantomData, ops::Range}; +use std::{fmt::Debug, marker::PhantomData}; use blockscout_metrics_tools::AggregateTimer; use chrono::{DateTime, Utc}; -use sea_orm::{prelude::DateTimeUtc, DatabaseConnection, DbErr}; +use sea_orm::{DatabaseConnection, DbErr}; use crate::{ data_source::{DataSource, UpdateContext}, + range::UniversalRange, types::{ConsistsOf, Timespan, TimespanValue}, UpdateError, }; @@ -22,7 +23,7 @@ pub struct LastValueLowerResolution(PhantomData<(DS, LowerRes)>); impl DataSource for LastValueLowerResolution where - LowerRes: Timespan + ConsistsOf + Eq + Send, + LowerRes: Timespan + ConsistsOf + Eq + Ord + Send, HigherRes: Clone, Value: Send + Debug, DS: DataSource>>, @@ -51,16 +52,12 @@ where async fn query_data( cx: &UpdateContext<'_>, - range: Option>, + range: UniversalRange>, dependency_data_fetch_timer: &mut AggregateTimer, ) -> Result { - let time_range_for_lower_res = range.map(extend_to_timespan_boundaries::); - let high_res_data = DS::query_data( - cx, - time_range_for_lower_res.clone(), - dependency_data_fetch_timer, - ) - .await?; + let time_range_for_lower_res = extend_to_timespan_boundaries::(range); + let high_res_data = + DS::query_data(cx, time_range_for_lower_res, dependency_data_fetch_timer).await?; Ok(reduce_each_timespan( high_res_data, |t| LowerRes::from_smaller(t.timespan.clone()), @@ -87,6 +84,7 @@ mod tests { data_source::{types::BlockscoutMigrations, DataSource, UpdateContext}, gettable_const, lines::PredefinedMockSource, + range::UniversalRange, tests::point_construction::{d_v_int, dt, w_v_int}, types::timespans::{DateValue, Week}, MissingDatePolicy, @@ -121,7 +119,7 @@ mod tests { force_full: false, }; assert_eq!( - MockSource::query_data(&context, None, &mut AggregateTimer::new()) + MockSource::query_data(&context, UniversalRange::full(), &mut AggregateTimer::new()) .await .unwrap(), vec![ @@ -132,9 +130,13 @@ mod tests { ] ); assert_eq!( - MockSourceWeekly::query_data(&context, None, &mut AggregateTimer::new()) - .await - .unwrap(), + MockSourceWeekly::query_data( + &context, + UniversalRange::full(), + &mut AggregateTimer::new() + ) + .await + .unwrap(), vec![w_v_int("2024-07-08", 3), w_v_int("2024-07-22", 1234),] ); } diff --git a/stats/stats/src/data_source/kinds/data_manipulation/resolutions/mod.rs b/stats/stats/src/data_source/kinds/data_manipulation/resolutions/mod.rs index 2ceb4f462..33db3f8d6 100644 --- a/stats/stats/src/data_source/kinds/data_manipulation/resolutions/mod.rs +++ b/stats/stats/src/data_source/kinds/data_manipulation/resolutions/mod.rs @@ -2,34 +2,35 @@ //! type/meaning. //! E.g. "weekly average block rewards" from "daily average block rewards". -use std::ops::{Range, RangeInclusive}; - use chrono::{DateTime, Utc}; -use crate::{types::Timespan, utils::exclusive_datetime_range_to_inclusive}; +use crate::{range::UniversalRange, types::Timespan}; pub mod average; pub mod last_value; pub mod sum; // Boundaries of resulting range - timespans that contain boundaries of date range -fn date_range_to_timespan(range: Range>) -> RangeInclusive { - let range = exclusive_datetime_range_to_inclusive(range); - let start_timespan = T::from_date(range.start().date_naive()); - let end_timespan = T::from_date(range.end().date_naive()); - start_timespan..=end_timespan +fn date_range_to_timespan(range: UniversalRange>) -> UniversalRange { + let (start, end_inclusive) = range.into_inclusive_pair(); + let start_timespan = start.map(|s| T::from_date(s.date_naive())); + let end_timespan = end_inclusive.map(|e| T::from_date(e.date_naive())); + (start_timespan..=end_timespan).into() } -pub fn extend_to_timespan_boundaries( - range: Range>, -) -> Range> { +pub fn extend_to_timespan_boundaries( + range: UniversalRange>, +) -> UniversalRange> { let timespan_range = date_range_to_timespan::(range); // start of timespan containing range start - let start: DateTime = timespan_range.start().saturating_start_timestamp(); + let (start, end) = timespan_range.into_inclusive_pair(); + let start = start.map(|s| s.saturating_start_timestamp()); // start of timespan following range end (to get exclusive range again) - let timespan_after_range = timespan_range.end().saturating_next_timespan(); - let end = timespan_after_range.saturating_start_timestamp(); - start..end + let end = end.map(|e| { + let timespan_after_range = e.saturating_next_timespan(); + timespan_after_range.saturating_start_timestamp() + }); + (start..end).into() } /// Produce vector of timespan data `LResPoint` from vector of smaller timespan data `HResPoint`. @@ -87,45 +88,59 @@ mod tests { assert_eq!( date_range_to_timespan::( - dt("2024-07-08T09:00:00").and_utc()..dt("2024-07-14T09:00:00").and_utc() - ), + (dt("2024-07-08T09:00:00").and_utc()..dt("2024-07-14T09:00:00").and_utc()).into() + ) + .try_into_inclusive() + .unwrap(), week_of("2024-07-08")..=week_of("2024-07-08") ); assert_eq!( date_range_to_timespan::( - dt("2024-07-08T09:00:00").and_utc()..dt("2024-07-14T23:59:59").and_utc() - ), + (dt("2024-07-08T09:00:00").and_utc()..dt("2024-07-14T23:59:59").and_utc()).into() + ) + .try_into_inclusive() + .unwrap(), week_of("2024-07-08")..=week_of("2024-07-08") ); assert_eq!( date_range_to_timespan::( - dt("2024-07-08T09:00:00").and_utc()..dt("2024-07-15T00:00:00").and_utc() - ), + (dt("2024-07-08T09:00:00").and_utc()..dt("2024-07-15T00:00:00").and_utc()).into() + ) + .try_into_inclusive() + .unwrap(), week_of("2024-07-08")..=week_of("2024-07-08") ); assert_eq!( date_range_to_timespan::( - dt("1995-12-31T09:00:00").and_utc()..dt("1995-12-31T23:59:60").and_utc() - ), + (dt("1995-12-31T09:00:00").and_utc()..dt("1995-12-31T23:59:60").and_utc()).into() + ) + .try_into_inclusive() + .unwrap(), week_of("1995-12-31")..=week_of("1995-12-31") ); assert_eq!( date_range_to_timespan::( - dt("1995-12-31T09:00:00").and_utc()..dt("1996-01-01T00:00:00").and_utc() - ), + (dt("1995-12-31T09:00:00").and_utc()..dt("1996-01-01T00:00:00").and_utc()).into() + ) + .try_into_inclusive() + .unwrap(), week_of("1995-12-31")..=week_of("1995-12-31") ); assert_eq!( date_range_to_timespan::( - dt("2024-07-08T09:00:00").and_utc()..dt("2024-07-15T00:00:01").and_utc() - ), + (dt("2024-07-08T09:00:00").and_utc()..dt("2024-07-15T00:00:01").and_utc()).into() + ) + .try_into_inclusive() + .unwrap(), week_of("2024-07-08")..=week_of("2024-07-15") ); assert_eq!( date_range_to_timespan::( - dt("1995-12-31T09:00:00").and_utc()..dt("1996-01-01T00:00:01").and_utc() - ), + (dt("1995-12-31T09:00:00").and_utc()..dt("1996-01-01T00:00:01").and_utc()).into() + ) + .try_into_inclusive() + .unwrap(), week_of("1995-12-31")..=week_of("1996-01-01") ); } diff --git a/stats/stats/src/data_source/kinds/data_manipulation/resolutions/sum.rs b/stats/stats/src/data_source/kinds/data_manipulation/resolutions/sum.rs index 399666510..67896788c 100644 --- a/stats/stats/src/data_source/kinds/data_manipulation/resolutions/sum.rs +++ b/stats/stats/src/data_source/kinds/data_manipulation/resolutions/sum.rs @@ -3,19 +3,16 @@ //! Intended for "new"/"delta" charts where change of something //! is presented. -use std::{ - fmt::Debug, - marker::PhantomData, - ops::{AddAssign, Range}, -}; +use std::{fmt::Debug, marker::PhantomData, ops::AddAssign}; use blockscout_metrics_tools::AggregateTimer; use chrono::{DateTime, Utc}; use rust_decimal::prelude::Zero; -use sea_orm::{prelude::DateTimeUtc, DatabaseConnection, DbErr}; +use sea_orm::{DatabaseConnection, DbErr}; use crate::{ data_source::{DataSource, UpdateContext}, + range::UniversalRange, types::{ConsistsOf, Timespan, TimespanValue}, UpdateError, }; @@ -27,7 +24,7 @@ pub struct SumLowerResolution(PhantomData<(DS, LowerRes)>); impl DataSource for SumLowerResolution where - LowerRes: Timespan + ConsistsOf + Eq + Debug + Send, + LowerRes: Timespan + ConsistsOf + Eq + Ord + Debug + Send, HigherRes: Clone + Debug, Value: AddAssign + Zero + Send + Debug, DS: DataSource>>, @@ -56,16 +53,12 @@ where async fn query_data( cx: &UpdateContext<'_>, - range: Option>, + range: UniversalRange>, dependency_data_fetch_timer: &mut AggregateTimer, ) -> Result { - let time_range_for_lower_res = range.map(extend_to_timespan_boundaries::); - let high_res_data = DS::query_data( - cx, - time_range_for_lower_res.clone(), - dependency_data_fetch_timer, - ) - .await?; + let time_range_for_lower_res = extend_to_timespan_boundaries::(range); + let high_res_data = + DS::query_data(cx, time_range_for_lower_res, dependency_data_fetch_timer).await?; Ok(reduce_each_timespan( high_res_data, |t| LowerRes::from_smaller(t.timespan.clone()), @@ -111,6 +104,7 @@ mod tests { data_source::{types::BlockscoutMigrations, DataSource, UpdateContext}, gettable_const, lines::PredefinedMockSource, + range::UniversalRange, tests::point_construction::{d_v_int, dt, w_v_int}, types::timespans::{DateValue, Week}, MissingDatePolicy, @@ -145,7 +139,7 @@ mod tests { force_full: false, }; assert_eq!( - MockSource::query_data(&context, None, &mut AggregateTimer::new()) + MockSource::query_data(&context, UniversalRange::full(), &mut AggregateTimer::new()) .await .unwrap(), vec![ @@ -156,9 +150,13 @@ mod tests { ] ); assert_eq!( - MockSourceWeekly::query_data(&context, None, &mut AggregateTimer::new()) - .await - .unwrap(), + MockSourceWeekly::query_data( + &context, + UniversalRange::full(), + &mut AggregateTimer::new() + ) + .await + .unwrap(), vec![w_v_int("2024-07-08", 4), w_v_int("2024-07-22", 1239),] ); } diff --git a/stats/stats/src/data_source/kinds/data_manipulation/sum_point.rs b/stats/stats/src/data_source/kinds/data_manipulation/sum_point.rs index 6cc68f018..e26345f5e 100644 --- a/stats/stats/src/data_source/kinds/data_manipulation/sum_point.rs +++ b/stats/stats/src/data_source/kinds/data_manipulation/sum_point.rs @@ -4,17 +4,18 @@ use std::{ marker::PhantomData, - ops::{AddAssign, Range}, + ops::AddAssign, }; use blockscout_metrics_tools::AggregateTimer; use chrono::{DateTime, Utc}; use rust_decimal::prelude::Zero; -use sea_orm::{prelude::DateTimeUtc, DatabaseConnection, DbErr}; +use sea_orm::{DatabaseConnection, DbErr}; use crate::{ data_processing::sum, data_source::{source::DataSource, UpdateContext}, + range::UniversalRange, types::{Timespan, TimespanValue}, UpdateError, }; @@ -58,12 +59,13 @@ where async fn query_data( cx: &UpdateContext<'_>, - _range: Option>, + _range: UniversalRange>, dependency_data_fetch_timer: &mut AggregateTimer, ) -> Result { // it's possible to not request full data range and use last accurate point; // can be updated to work similarly to cumulative - let full_data = DS::query_data(cx, None, dependency_data_fetch_timer).await?; + let full_data = + DS::query_data(cx, UniversalRange::full(), dependency_data_fetch_timer).await?; tracing::debug!(points_len = full_data.len(), "calculating sum"); let zero = Value::zero(); sum::(&full_data, zero) diff --git a/stats/stats/src/data_source/kinds/local_db/mod.rs b/stats/stats/src/data_source/kinds/local_db/mod.rs index 26f84f0e5..b38843df7 100644 --- a/stats/stats/src/data_source/kinds/local_db/mod.rs +++ b/stats/stats/src/data_source/kinds/local_db/mod.rs @@ -9,7 +9,7 @@ //! Charts are intended to be such persisted sources, //! because their data is directly retreived from the database (on requests). -use std::{fmt::Debug, marker::PhantomData, ops::Range, time::Duration}; +use std::{fmt::Debug, marker::PhantomData, time::Duration}; use blockscout_metrics_tools::AggregateTimer; use chrono::{DateTime, SubsecRound, Utc}; @@ -24,7 +24,7 @@ use parameters::{ }, DefaultCreate, DefaultQueryLast, DefaultQueryVec, QueryLastWithEstimationFallback, }; -use sea_orm::{prelude::DateTimeUtc, DatabaseConnection, DbErr}; +use sea_orm::{DatabaseConnection, DbErr}; use crate::{ charts::{ @@ -33,7 +33,9 @@ use crate::{ ChartProperties, Named, }, data_source::{DataSource, UpdateContext}, - metrics, UpdateError, + metrics, + range::UniversalRange, + UpdateError, }; use super::auxiliary::PartialCumulative; @@ -262,13 +264,13 @@ where async fn query_data( cx: &UpdateContext<'_>, - range: Option>, + range: UniversalRange>, dependency_data_fetch_timer: &mut AggregateTimer, ) -> Result { let _timer = dependency_data_fetch_timer.start_interval(); // maybe add `fill_missing_dates` parameter to current function as well in the future // to get rid of "Note" in the `DataSource`'s method documentation - Query::query_data(cx, range, false).await + Query::query_data(cx, range, None, false).await } } diff --git a/stats/stats/src/data_source/kinds/local_db/parameter_traits.rs b/stats/stats/src/data_source/kinds/local_db/parameter_traits.rs index 93d8ac4ca..4f3b88bf0 100644 --- a/stats/stats/src/data_source/kinds/local_db/parameter_traits.rs +++ b/stats/stats/src/data_source/kinds/local_db/parameter_traits.rs @@ -1,14 +1,15 @@ -use std::{future::Future, marker::Send, ops::Range}; +use std::{future::Future, marker::Send}; use blockscout_metrics_tools::AggregateTimer; use chrono::{DateTime, Utc}; -use sea_orm::{prelude::DateTimeUtc, DatabaseConnection, DbErr}; +use sea_orm::{DatabaseConnection, DbErr}; use crate::{ charts::db_interaction::write::set_last_updated_at, data_source::{DataSource, UpdateContext}, + range::UniversalRange, types::TimespanValue, - UpdateError, + RequestedPointsLimit, UpdateError, }; /// In most cases, [`super::DefaultCreate`] is enough. @@ -61,7 +62,8 @@ pub trait QueryBehaviour { /// Retrieve chart data from local storage. fn query_data( cx: &UpdateContext<'_>, - range: Option>, + range: UniversalRange>, + points_limit: Option, fill_missing_dates: bool, ) -> impl Future> + Send; } diff --git a/stats/stats/src/data_source/kinds/local_db/parameters/query.rs b/stats/stats/src/data_source/kinds/local_db/parameters/query.rs index 707cadcf3..68a38ae9d 100644 --- a/stats/stats/src/data_source/kinds/local_db/parameters/query.rs +++ b/stats/stats/src/data_source/kinds/local_db/parameters/query.rs @@ -1,14 +1,15 @@ -use std::{fmt::Debug, marker::PhantomData, ops::Range}; +use std::{fmt::Debug, marker::PhantomData}; -use sea_orm::{prelude::DateTimeUtc, DatabaseConnection}; +use chrono::{DateTime, Utc}; +use sea_orm::DatabaseConnection; use crate::{ charts::db_interaction::read::get_counter_data, data_source::{kinds::local_db::parameter_traits::QueryBehaviour, UpdateContext}, get_line_chart_data, + range::UniversalRange, types::{timespans::DateValue, ExtendedTimespanValue, Timespan}, - utils::exclusive_datetime_range_to_inclusive, - ChartProperties, UpdateError, + ChartProperties, RequestedPointsLimit, UpdateError, }; /// Usually the choice for line charts @@ -28,14 +29,14 @@ where /// Expects metadata to be consistent with stored data async fn query_data( cx: &UpdateContext<'_>, - range: Option>, + range: UniversalRange>, + points_limit: Option, fill_missing_dates: bool, ) -> Result { // In DB we store data with date precision. Also, `get_line_chart_data` // works with inclusive range. Therefore, we need to convert the range and // get date without time. - let range = range.map(exclusive_datetime_range_to_inclusive); - let (start, end) = range.map(|r| r.into_inner()).unzip(); + let (start, end) = range.into_inclusive_pair(); // At the same time, update-time relevance for local charts // is achieved while requesting remote source data. @@ -52,7 +53,7 @@ where &C::name(), start, end, - None, + points_limit, C::missing_date_policy(), fill_missing_dates, C::approximate_trailing_points(), @@ -70,7 +71,8 @@ impl QueryBehaviour for DefaultQueryLast { async fn query_data( cx: &UpdateContext<'_>, - _range: Option>, + _range: UniversalRange>, + _points_limit: Option, _fill_missing_dates: bool, ) -> Result { let value = get_counter_data( @@ -107,7 +109,8 @@ where async fn query_data( cx: &UpdateContext<'_>, - _range: Option>, + _range: UniversalRange>, + _points_limit: Option, _fill_missing_dates: bool, ) -> Result { let value = get_counter_data( @@ -193,7 +196,10 @@ mod tests { assert_eq!( expected_estimate(), QueryLastWithEstimationFallback::::query_data( - &cx, None, true + &cx, + UniversalRange::full(), + None, + true ) .await .unwrap() diff --git a/stats/stats/src/data_source/kinds/local_db/parameters/update/batching/mod.rs b/stats/stats/src/data_source/kinds/local_db/parameters/update/batching/mod.rs index fc1195f4a..cbfa24996 100644 --- a/stats/stats/src/data_source/kinds/local_db/parameters/update/batching/mod.rs +++ b/stats/stats/src/data_source/kinds/local_db/parameters/update/batching/mod.rs @@ -17,6 +17,7 @@ use crate::{ types::Get, UpdateContext, }, + range::UniversalRange, types::{ExtendedTimespanValue, Timespan, TimespanDuration, TimespanValue}, ChartProperties, UpdateError, }; @@ -131,7 +132,11 @@ where let previous_step_last_point_timespan = this_step_start.saturating_previous_timespan(); let last_point_range_values = Query::query_data( cx, - Some(previous_step_last_point_timespan.clone().into_time_range()), + previous_step_last_point_timespan + .clone() + .into_time_range() + .into(), + None, false, ) .await?; @@ -168,11 +173,11 @@ where Resolution: Timespan, BatchStep: BatchStepBehaviour, { - let query_range = range.into_date_time_range(); + let query_range: UniversalRange<_> = range.into_date_time_range().into(); let main_data = - MainDep::query_data(cx, Some(query_range.clone()), dependency_data_fetch_timer).await?; + MainDep::query_data(cx, query_range.clone(), dependency_data_fetch_timer).await?; let resolution_data = - ResolutionDep::query_data(cx, Some(query_range), dependency_data_fetch_timer).await?; + ResolutionDep::query_data(cx, query_range, dependency_data_fetch_timer).await?; let found = BatchStep::batch_update_values_step_with( cx.db, chart_id, diff --git a/stats/stats/src/data_source/kinds/local_db/parameters/update/point.rs b/stats/stats/src/data_source/kinds/local_db/parameters/update/point.rs index 1bba427a6..20a3f8e78 100644 --- a/stats/stats/src/data_source/kinds/local_db/parameters/update/point.rs +++ b/stats/stats/src/data_source/kinds/local_db/parameters/update/point.rs @@ -5,6 +5,7 @@ use blockscout_metrics_tools::AggregateTimer; use crate::{ charts::db_interaction::write::insert_data_many, data_source::{kinds::local_db::UpdateBehaviour, DataSource, UpdateContext}, + range::UniversalRange, types::{Timespan, TimespanValue}, UpdateError, }; @@ -25,7 +26,7 @@ where remote_fetch_timer: &mut AggregateTimer, ) -> Result<(), UpdateError> { // range doesn't make sense there; thus is not used - let data = MainDep::query_data(cx, None, remote_fetch_timer).await?; + let data = MainDep::query_data(cx, UniversalRange::full(), remote_fetch_timer).await?; let value = data.active_model(chart_id, Some(min_blockscout_block)); insert_data_many(cx.db, vec![value]) .await diff --git a/stats/stats/src/data_source/kinds/remote_db/mod.rs b/stats/stats/src/data_source/kinds/remote_db/mod.rs index 0a68f44c5..beb69e8d5 100644 --- a/stats/stats/src/data_source/kinds/remote_db/mod.rs +++ b/stats/stats/src/data_source/kinds/remote_db/mod.rs @@ -21,7 +21,6 @@ mod query; use std::{ future::Future, marker::{PhantomData, Send}, - ops::Range, }; use blockscout_metrics_tools::AggregateTimer; @@ -30,6 +29,7 @@ use sea_orm::{DatabaseConnection, DbErr}; use crate::{ data_source::{source::DataSource, types::UpdateContext}, + range::UniversalRange, UpdateError, }; @@ -47,7 +47,7 @@ pub trait RemoteQueryBehaviour { /// Retrieve chart data from remote storage. fn query_data( cx: &UpdateContext<'_>, - range: Option>>, + range: UniversalRange>, ) -> impl Future> + Send; } @@ -69,7 +69,7 @@ impl DataSource for RemoteDatabaseSource { async fn query_data( cx: &UpdateContext<'_>, - range: Option>>, + range: UniversalRange>, remote_fetch_timer: &mut AggregateTimer, ) -> Result<::Output, UpdateError> { let _interval = remote_fetch_timer.start_interval(); diff --git a/stats/stats/src/data_source/kinds/remote_db/query/all.rs b/stats/stats/src/data_source/kinds/remote_db/query/all.rs index d7fdd777a..de30f0ef5 100644 --- a/stats/stats/src/data_source/kinds/remote_db/query/all.rs +++ b/stats/stats/src/data_source/kinds/remote_db/query/all.rs @@ -3,13 +3,15 @@ use std::{ ops::Range, }; -use sea_orm::{prelude::DateTimeUtc, FromQueryResult, Statement}; +use chrono::{DateTime, Utc}; +use sea_orm::{FromQueryResult, Statement}; use crate::{ data_source::{ kinds::remote_db::RemoteQueryBehaviour, types::{BlockscoutMigrations, UpdateContext}, }, + range::{data_source_query_range_to_db_statement_range, UniversalRange}, types::TimespanValue, UpdateError, }; @@ -17,7 +19,7 @@ use crate::{ pub trait StatementFromRange { /// `completed_migrations` can be used for selecting more optimal query fn get_statement( - range: Option>, + range: Option>>, completed_migrations: &BlockscoutMigrations, ) -> Statement; } @@ -28,27 +30,35 @@ pub trait StatementFromRange { /// `P` - Type of point to retrieve within query. /// `DateValue` can be used to avoid parsing the values, /// but `DateValue` or other types can be useful sometimes. -pub struct PullAllWithAndSort(PhantomData<(S, Resolution, Value)>) +pub struct PullAllWithAndSort( + PhantomData<(S, Resolution, Value, AllRangeSource)>, +) where S: StatementFromRange, Resolution: Ord + Send, Value: Send, - TimespanValue: FromQueryResult; + TimespanValue: FromQueryResult, + AllRangeSource: RemoteQueryBehaviour>>; -impl RemoteQueryBehaviour for PullAllWithAndSort +impl RemoteQueryBehaviour + for PullAllWithAndSort where S: StatementFromRange, Resolution: Ord + Send, Value: Send, TimespanValue: FromQueryResult, + AllRangeSource: RemoteQueryBehaviour>>, { type Output = Vec>; async fn query_data( cx: &UpdateContext<'_>, - range: Option>, + range: UniversalRange>, ) -> Result>, UpdateError> { - let query = S::get_statement(range, &cx.blockscout_applied_migrations); + // to not overcomplicate the queries + let query_range = + data_source_query_range_to_db_statement_range::(cx, range).await?; + let query = S::get_statement(query_range, &cx.blockscout_applied_migrations); let mut data = TimespanValue::::find_by_statement(query) .all(cx.blockscout) .await diff --git a/stats/stats/src/data_source/kinds/remote_db/query/each.rs b/stats/stats/src/data_source/kinds/remote_db/query/each.rs index 76285aaa3..e1c2c4b91 100644 --- a/stats/stats/src/data_source/kinds/remote_db/query/each.rs +++ b/stats/stats/src/data_source/kinds/remote_db/query/each.rs @@ -12,7 +12,7 @@ use crate::{ kinds::remote_db::RemoteQueryBehaviour, types::{BlockscoutMigrations, UpdateContext}, }, - exclusive_datetime_range_to_inclusive, + range::{exclusive_range_to_inclusive, UniversalRange}, types::{Timespan, TimespanValue}, UpdateError, }; @@ -50,12 +50,13 @@ where async fn query_data( cx: &UpdateContext<'_>, - range: Option>>, + range: UniversalRange>, ) -> Result>, UpdateError> { - let query_range = if let Some(r) = range { + let query_range = if let Some(r) = range.clone().try_into_exclusive() { r } else { - AllRangeSource::query_data(cx, None).await? + let whole_range = AllRangeSource::query_data(cx, UniversalRange::full()).await?; + range.into_exclusive_with_backup(whole_range) }; let points = split_time_range_into_resolution_points::(query_range); let mut collected_data = Vec::with_capacity(points.len()); @@ -75,7 +76,7 @@ where } fn resolution_from_range(range: Range>) -> R { - let range = exclusive_datetime_range_to_inclusive(range); + let range = exclusive_range_to_inclusive(range); let res = R::from_date(range.start().date_naive()); let res_verify = R::from_date(range.end().date_naive()); if res_verify != res { diff --git a/stats/stats/src/data_source/kinds/remote_db/query/one.rs b/stats/stats/src/data_source/kinds/remote_db/query/one.rs index 54fa365fb..96f781e64 100644 --- a/stats/stats/src/data_source/kinds/remote_db/query/one.rs +++ b/stats/stats/src/data_source/kinds/remote_db/query/one.rs @@ -1,15 +1,14 @@ -use std::{ - marker::{PhantomData, Send}, - ops::Range, -}; +use std::marker::{PhantomData, Send}; -use sea_orm::{prelude::DateTimeUtc, FromQueryResult, Statement}; +use chrono::{DateTime, Utc}; +use sea_orm::{FromQueryResult, Statement}; use crate::{ data_source::{ kinds::remote_db::RemoteQueryBehaviour, types::{BlockscoutMigrations, UpdateContext}, }, + range::UniversalRange, types::TimespanValue, UpdateError, }; @@ -42,7 +41,7 @@ where async fn query_data( cx: &UpdateContext<'_>, - _range: Option>, + _range: UniversalRange>, ) -> Result, UpdateError> { let query = S::get_statement(&cx.blockscout_applied_migrations); let data = TimespanValue::::find_by_statement(query) diff --git a/stats/stats/src/data_source/source.rs b/stats/stats/src/data_source/source.rs index 2f3fbb234..4c72e4760 100644 --- a/stats/stats/src/data_source/source.rs +++ b/stats/stats/src/data_source/source.rs @@ -1,13 +1,13 @@ -use std::{collections::HashSet, future::Future, marker::Send, ops::Range}; +use std::{collections::HashSet, future::Future, marker::Send}; use blockscout_metrics_tools::AggregateTimer; -use chrono::Utc; +use chrono::{DateTime, Utc}; use futures::{future::BoxFuture, FutureExt}; -use sea_orm::{prelude::DateTimeUtc, DatabaseConnection, DbErr}; +use sea_orm::{DatabaseConnection, DbErr}; use tracing::instrument; use tynm::type_name; -use crate::UpdateError; +use crate::{range::UniversalRange, UpdateError}; use super::types::UpdateContext; @@ -163,7 +163,7 @@ pub trait DataSource { /// to call [`DataSource::update_recursively`] beforehand. fn query_data( cx: &UpdateContext<'_>, - range: Option>, + range: UniversalRange>, dependency_data_fetch_timer: &mut AggregateTimer, ) -> impl Future> + Send; } @@ -208,7 +208,7 @@ impl DataSource for () { async fn query_data( _cx: &UpdateContext<'_>, - _range: Option>, + _range: UniversalRange>, _remote_fetch_timer: &mut AggregateTimer, ) -> Result { Ok(()) @@ -249,7 +249,7 @@ macro_rules! impl_data_source_for_tuple { async fn query_data( cx: &UpdateContext<'_>, - range: Option>, + range: UniversalRange>, remote_fetch_timer: &mut AggregateTimer, ) -> Result { Ok(( diff --git a/stats/stats/src/data_source/tests.rs b/stats/stats/src/data_source/tests.rs index 839333976..887646a50 100644 --- a/stats/stats/src/data_source/tests.rs +++ b/stats/stats/src/data_source/tests.rs @@ -27,6 +27,7 @@ use super::{ types::UpdateParameters, }; use crate::{ + charts::db_interaction::read::QueryAllBlockTimestampRange, construct_update_group, data_source::{ kinds::local_db::parameters::update::batching::parameters::PassVecStep, @@ -127,8 +128,9 @@ impl StatementFromRange for NewContractsQuery { } } -pub type NewContractsRemote = - RemoteDatabaseSource>; +pub type NewContractsRemote = RemoteDatabaseSource< + PullAllWithAndSort, +>; pub struct NewContractsChartProperties; diff --git a/stats/stats/src/data_source/types.rs b/stats/stats/src/data_source/types.rs index 6f9ae5c5e..56e3b3486 100644 --- a/stats/stats/src/data_source/types.rs +++ b/stats/stats/src/data_source/types.rs @@ -1,4 +1,3 @@ -use anyhow::Context; use blockscout_db::entity::migrations_status; use chrono::Utc; use sea_orm::{DatabaseConnection, DbErr, EntityTrait, FromQueryResult, QueryOrder, Statement}; diff --git a/stats/stats/src/lib.rs b/stats/stats/src/lib.rs index b4d49407c..546af418f 100644 --- a/stats/stats/src/lib.rs +++ b/stats/stats/src/lib.rs @@ -3,9 +3,10 @@ pub mod data_processing; pub mod data_source; pub mod metrics; mod missing_date; +pub mod range; pub mod update_group; pub mod update_groups; -pub(crate) mod utils; +pub mod utils; #[cfg(any(feature = "test-utils", test))] pub mod tests; @@ -16,10 +17,9 @@ pub use migration; pub use charts::{ counters, db_interaction::read::{ - get_line_chart_data, get_raw_counters, ApproxUnsignedDiff, ReadError, RequestedPointsLimit, + get_line_chart_data, get_raw_counters, ApproxUnsignedDiff, QueryAllBlockTimestampRange, + ReadError, RequestedPointsLimit, }, lines, query_dispatch, types, ChartKey, ChartObject, ChartProperties, ChartPropertiesObject, MissingDatePolicy, Named, ResolutionKind, UpdateError, }; - -pub use utils::exclusive_datetime_range_to_inclusive; diff --git a/stats/stats/src/range.rs b/stats/stats/src/range.rs index 4156ceb68..904c7d260 100644 --- a/stats/stats/src/range.rs +++ b/stats/stats/src/range.rs @@ -315,8 +315,8 @@ where #[cfg(test)] mod tests { use super::*; - use crate::{tests::point_construction::*, types::timespans::Week}; - use chrono::{NaiveDate, NaiveDateTime}; + use crate::tests::point_construction::*; + use chrono::NaiveDate; use pretty_assertions::assert_eq; #[test] diff --git a/stats/stats/src/update_group.rs b/stats/stats/src/update_group.rs index 7787c2f4a..076f36098 100644 --- a/stats/stats/src/update_group.rs +++ b/stats/stats/src/update_group.rs @@ -197,7 +197,10 @@ pub trait UpdateGroup: core::fmt::Debug { /// ## Example /// /// ```rust -/// # use stats::{ChartProperties, Named, construct_update_group, types::timespans::DateValue, UpdateError}; +/// # use stats::{ +/// # QueryAllBlockTimestampRange, construct_update_group, +/// # types::timespans::DateValue, ChartProperties, Named, UpdateError, +/// # }; /// # use stats::data_source::{ /// # kinds::{ /// # local_db::{DirectVecLocalDbChartSource, parameters::update::batching::parameters::Batch30Days}, @@ -219,7 +222,7 @@ pub trait UpdateGroup: core::fmt::Debug { /// } /// } /// -/// type DummyRemote = RemoteDatabaseSource>; +/// type DummyRemote = RemoteDatabaseSource>; /// /// struct DummyChartProperties; /// diff --git a/stats/stats/src/utils.rs b/stats/stats/src/utils.rs index 185da50e4..54a6c1f59 100644 --- a/stats/stats/src/utils.rs +++ b/stats/stats/src/utils.rs @@ -1,28 +1,18 @@ //! Common utilities used across statistics -use std::ops::{Range, RangeInclusive}; +use std::ops::Range; -use chrono::{NaiveDate, NaiveTime}; -use sea_orm::{prelude::DateTimeUtc, Value}; +use chrono::{DateTime, NaiveDate, NaiveTime, Utc}; +use sea_orm::Value; // this const is not public in `chrono` for some reason pub const NANOS_PER_SEC: i32 = 1_000_000_000; -pub fn day_start(date: &NaiveDate) -> DateTimeUtc { +pub fn day_start(date: &NaiveDate) -> DateTime { date.and_time(NaiveTime::from_hms_opt(0, 0, 0).expect("correct time")) .and_utc() } -pub fn exclusive_datetime_range_to_inclusive(r: Range) -> RangeInclusive { - // subtract the smallest unit of time to get semantically the same range - // but inclusive - let new_end = r - .end - .checked_sub_signed(chrono::Duration::nanoseconds(1)) - .unwrap_or(DateTimeUtc::MIN_UTC); // saturating sub - r.start..=new_end -} - /// Used inside [`sql_with_range_filter_opt`] /// /// `filter_arg_number_start = len(arg)+1 // (length of other args + 1)` @@ -32,7 +22,7 @@ pub fn exclusive_datetime_range_to_inclusive(r: Range) -> RangeIncl /// Vec should be appended to the args. /// String should be inserted in places for filter. pub(crate) fn produce_filter_and_values( - range: Option>, + range: Option>>, filter_by: &str, filter_arg_number_start: usize, ) -> (String, Vec) { From 7e7df45a401617121701b893bbe6f8c86083ec5e Mon Sep 17 00:00:00 2001 From: Kirill Ivanov Date: Mon, 2 Dec 2024 22:15:17 +0900 Subject: [PATCH 14/41] war on DateTimeUtc --- stats/stats/src/charts/counters/total_blocks.rs | 1 - stats/stats/src/charts/db_interaction/write.rs | 2 +- stats/stats/src/charts/lines/active_accounts.rs | 6 +++--- stats/stats/src/charts/lines/average_block_rewards.rs | 5 +++-- stats/stats/src/charts/lines/average_block_size.rs | 5 +++-- stats/stats/src/charts/lines/average_gas_limit.rs | 5 +++-- stats/stats/src/charts/lines/average_gas_price.rs | 5 +++-- stats/stats/src/charts/lines/average_txn_fee.rs | 5 +++-- stats/stats/src/charts/lines/gas_used_growth.rs | 6 ++++-- stats/stats/src/charts/lines/native_coin_supply.rs | 5 +++-- stats/stats/src/charts/lines/new_accounts.rs | 6 +++--- stats/stats/src/charts/lines/new_block_rewards.rs | 5 +++-- stats/stats/src/charts/lines/new_blocks.rs | 5 +++-- stats/stats/src/charts/lines/new_contracts.rs | 5 +++-- .../src/charts/lines/new_native_coin_transfers.rs | 5 +++-- stats/stats/src/charts/lines/new_txns.rs | 5 +++-- stats/stats/src/charts/lines/new_verified_contracts.rs | 5 +++-- stats/stats/src/charts/lines/txns_fee.rs | 5 +++-- stats/stats/src/charts/lines/txns_success_rate.rs | 5 +++-- stats/stats/src/data_source/tests.rs | 4 ++-- stats/stats/src/update_group.rs | 4 +--- stats/stats/src/utils.rs | 10 +++++----- 22 files changed, 61 insertions(+), 48 deletions(-) diff --git a/stats/stats/src/charts/counters/total_blocks.rs b/stats/stats/src/charts/counters/total_blocks.rs index b1a0b37dc..f20820b7d 100644 --- a/stats/stats/src/charts/counters/total_blocks.rs +++ b/stats/stats/src/charts/counters/total_blocks.rs @@ -1,4 +1,3 @@ - use crate::{ charts::db_interaction::read::get_estimated_table_rows, data_source::{ diff --git a/stats/stats/src/charts/db_interaction/write.rs b/stats/stats/src/charts/db_interaction/write.rs index d70dd38a5..f5706889d 100644 --- a/stats/stats/src/charts/db_interaction/write.rs +++ b/stats/stats/src/charts/db_interaction/write.rs @@ -1,6 +1,6 @@ use chrono::{DateTime, Offset, TimeZone}; use entity::{chart_data, charts, sea_orm_active_enums::ChartType}; -use sea_orm::{prelude::*, sea_query, ConnectionTrait, Set, Unchanged}; +use sea_orm::{prelude::*, sea_query, Set, Unchanged}; use crate::charts::ChartKey; diff --git a/stats/stats/src/charts/lines/active_accounts.rs b/stats/stats/src/charts/lines/active_accounts.rs index 19a91b285..86172eba8 100644 --- a/stats/stats/src/charts/lines/active_accounts.rs +++ b/stats/stats/src/charts/lines/active_accounts.rs @@ -17,15 +17,15 @@ use crate::{ ChartProperties, Named, }; -use chrono::NaiveDate; +use chrono::{DateTime, NaiveDate, Utc}; use entity::sea_orm_active_enums::ChartType; -use sea_orm::{prelude::*, DbBackend, Statement}; +use sea_orm::{DbBackend, Statement}; pub struct ActiveAccountsStatement; impl StatementFromRange for ActiveAccountsStatement { fn get_statement( - range: Option>, + range: Option>>, completed_migrations: &BlockscoutMigrations, ) -> Statement { if completed_migrations.denormalization { diff --git a/stats/stats/src/charts/lines/average_block_rewards.rs b/stats/stats/src/charts/lines/average_block_rewards.rs index ff55b9b27..9b6c849b1 100644 --- a/stats/stats/src/charts/lines/average_block_rewards.rs +++ b/stats/stats/src/charts/lines/average_block_rewards.rs @@ -25,8 +25,9 @@ use crate::{ }; use chrono::NaiveDate; +use chrono::{DateTime, Utc}; use entity::sea_orm_active_enums::ChartType; -use sea_orm::{prelude::*, DbBackend, Statement}; +use sea_orm::{DbBackend, Statement}; use super::{NewBlockRewardsInt, NewBlockRewardsMonthlyInt}; @@ -35,7 +36,7 @@ const ETH: i64 = 1_000_000_000_000_000_000; pub struct AverageBlockRewardsQuery; impl StatementFromRange for AverageBlockRewardsQuery { - fn get_statement(range: Option>, _: &BlockscoutMigrations) -> Statement { + fn get_statement(range: Option>>, _: &BlockscoutMigrations) -> Statement { sql_with_range_filter_opt!( DbBackend::Postgres, r#" diff --git a/stats/stats/src/charts/lines/average_block_size.rs b/stats/stats/src/charts/lines/average_block_size.rs index b637142d4..06d216ef1 100644 --- a/stats/stats/src/charts/lines/average_block_size.rs +++ b/stats/stats/src/charts/lines/average_block_size.rs @@ -25,15 +25,16 @@ use crate::{ }; use chrono::NaiveDate; +use chrono::{DateTime, Utc}; use entity::sea_orm_active_enums::ChartType; -use sea_orm::{prelude::*, DbBackend, Statement}; +use sea_orm::{DbBackend, Statement}; use super::new_blocks::{NewBlocksInt, NewBlocksMonthlyInt}; pub struct AverageBlockSizeStatement; impl StatementFromRange for AverageBlockSizeStatement { - fn get_statement(range: Option>, _: &BlockscoutMigrations) -> Statement { + fn get_statement(range: Option>>, _: &BlockscoutMigrations) -> Statement { sql_with_range_filter_opt!( DbBackend::Postgres, r#" diff --git a/stats/stats/src/charts/lines/average_gas_limit.rs b/stats/stats/src/charts/lines/average_gas_limit.rs index a397123fb..291ccf940 100644 --- a/stats/stats/src/charts/lines/average_gas_limit.rs +++ b/stats/stats/src/charts/lines/average_gas_limit.rs @@ -25,15 +25,16 @@ use crate::{ }; use chrono::NaiveDate; +use chrono::{DateTime, Utc}; use entity::sea_orm_active_enums::ChartType; -use sea_orm::{prelude::*, DbBackend, Statement}; +use sea_orm::{DbBackend, Statement}; use super::new_blocks::{NewBlocksInt, NewBlocksMonthlyInt}; pub struct AverageGasLimitStatement; impl StatementFromRange for AverageGasLimitStatement { - fn get_statement(range: Option>, _: &BlockscoutMigrations) -> Statement { + fn get_statement(range: Option>>, _: &BlockscoutMigrations) -> Statement { sql_with_range_filter_opt!( DbBackend::Postgres, r#" diff --git a/stats/stats/src/charts/lines/average_gas_price.rs b/stats/stats/src/charts/lines/average_gas_price.rs index 5b5f2b6e7..80d380765 100644 --- a/stats/stats/src/charts/lines/average_gas_price.rs +++ b/stats/stats/src/charts/lines/average_gas_price.rs @@ -25,8 +25,9 @@ use crate::{ }; use chrono::NaiveDate; +use chrono::{DateTime, Utc}; use entity::sea_orm_active_enums::ChartType; -use sea_orm::{prelude::*, DbBackend, Statement}; +use sea_orm::{DbBackend, Statement}; use super::new_txns::{NewTxnsInt, NewTxnsMonthlyInt}; @@ -36,7 +37,7 @@ pub struct AverageGasPriceStatement; impl StatementFromRange for AverageGasPriceStatement { fn get_statement( - range: Option>, + range: Option>>, completed_migrations: &BlockscoutMigrations, ) -> Statement { if completed_migrations.denormalization { diff --git a/stats/stats/src/charts/lines/average_txn_fee.rs b/stats/stats/src/charts/lines/average_txn_fee.rs index 6b678cfe2..431d8a2c4 100644 --- a/stats/stats/src/charts/lines/average_txn_fee.rs +++ b/stats/stats/src/charts/lines/average_txn_fee.rs @@ -27,8 +27,9 @@ use crate::{ }; use chrono::NaiveDate; +use chrono::{DateTime, Utc}; use entity::sea_orm_active_enums::ChartType; -use sea_orm::{prelude::*, DbBackend, Statement}; +use sea_orm::{DbBackend, Statement}; use super::new_txns::{NewTxnsInt, NewTxnsMonthlyInt}; @@ -38,7 +39,7 @@ pub struct AverageTxnFeeStatement; impl StatementFromRange for AverageTxnFeeStatement { fn get_statement( - range: Option>, + range: Option>>, completed_migrations: &BlockscoutMigrations, ) -> Statement { if completed_migrations.denormalization { diff --git a/stats/stats/src/charts/lines/gas_used_growth.rs b/stats/stats/src/charts/lines/gas_used_growth.rs index 208f64d66..0421a4a44 100644 --- a/stats/stats/src/charts/lines/gas_used_growth.rs +++ b/stats/stats/src/charts/lines/gas_used_growth.rs @@ -25,13 +25,15 @@ use crate::{ }; use chrono::NaiveDate; +use chrono::{DateTime, Utc}; use entity::sea_orm_active_enums::ChartType; -use sea_orm::{prelude::*, DbBackend, Statement}; +use rust_decimal::Decimal; +use sea_orm::{DbBackend, Statement}; pub struct GasUsedPartialStatement; impl StatementFromRange for GasUsedPartialStatement { - fn get_statement(range: Option>, _: &BlockscoutMigrations) -> Statement { + fn get_statement(range: Option>>, _: &BlockscoutMigrations) -> Statement { sql_with_range_filter_opt!( DbBackend::Postgres, r#" diff --git a/stats/stats/src/charts/lines/native_coin_supply.rs b/stats/stats/src/charts/lines/native_coin_supply.rs index 5b537c59d..b3ff3188b 100644 --- a/stats/stats/src/charts/lines/native_coin_supply.rs +++ b/stats/stats/src/charts/lines/native_coin_supply.rs @@ -24,15 +24,16 @@ use crate::{ }; use chrono::NaiveDate; +use chrono::{DateTime, Utc}; use entity::sea_orm_active_enums::ChartType; -use sea_orm::{prelude::*, DbBackend, Statement}; +use sea_orm::{DbBackend, Statement}; const ETH: i64 = 1_000_000_000_000_000_000; pub struct NativeCoinSupplyStatement; impl StatementFromRange for NativeCoinSupplyStatement { - fn get_statement(range: Option>, _: &BlockscoutMigrations) -> Statement { + fn get_statement(range: Option>>, _: &BlockscoutMigrations) -> Statement { let day_range: Option> = range.map(|r| { let Range { start, end } = r; // chart is off anyway, so shouldn't be a big deal diff --git a/stats/stats/src/charts/lines/new_accounts.rs b/stats/stats/src/charts/lines/new_accounts.rs index 2266e0a04..851f4b113 100644 --- a/stats/stats/src/charts/lines/new_accounts.rs +++ b/stats/stats/src/charts/lines/new_accounts.rs @@ -29,17 +29,17 @@ use crate::{ use chrono::{DateTime, NaiveDate, Utc}; use entity::sea_orm_active_enums::ChartType; -use sea_orm::{prelude::*, DbBackend, FromQueryResult, Statement}; +use sea_orm::{DbBackend, FromQueryResult, Statement}; pub struct NewAccountsStatement; impl StatementFromRange for NewAccountsStatement { fn get_statement( - range: Option>, + range: Option>>, completed_migrations: &BlockscoutMigrations, ) -> Statement { // `MIN_UTC` does not fit into postgres' timestamp. Unix epoch start should be enough - let min_timestamp = DateTimeUtc::UNIX_EPOCH; + let min_timestamp = DateTime::::UNIX_EPOCH; // All transactions from the beginning must be considered to calculate new accounts correctly. // E.g. if account was first active both before `range.start()` and within the range, // we don't want to count it within the range (as it's not a *new* account). diff --git a/stats/stats/src/charts/lines/new_block_rewards.rs b/stats/stats/src/charts/lines/new_block_rewards.rs index 402537b37..365af1aa8 100644 --- a/stats/stats/src/charts/lines/new_block_rewards.rs +++ b/stats/stats/src/charts/lines/new_block_rewards.rs @@ -27,13 +27,14 @@ use crate::{ }; use chrono::NaiveDate; +use chrono::{DateTime, Utc}; use entity::sea_orm_active_enums::ChartType; -use sea_orm::{prelude::*, DbBackend, Statement}; +use sea_orm::{DbBackend, Statement}; pub struct NewBlockRewardsStatement; impl StatementFromRange for NewBlockRewardsStatement { - fn get_statement(range: Option>, _: &BlockscoutMigrations) -> Statement { + fn get_statement(range: Option>>, _: &BlockscoutMigrations) -> Statement { sql_with_range_filter_opt!( DbBackend::Postgres, r#" diff --git a/stats/stats/src/charts/lines/new_blocks.rs b/stats/stats/src/charts/lines/new_blocks.rs index 414e70d22..bbef63a07 100644 --- a/stats/stats/src/charts/lines/new_blocks.rs +++ b/stats/stats/src/charts/lines/new_blocks.rs @@ -25,13 +25,14 @@ use crate::{ }; use chrono::NaiveDate; +use chrono::{DateTime, Utc}; use entity::sea_orm_active_enums::ChartType; -use sea_orm::{prelude::*, DbBackend, Statement}; +use sea_orm::{DbBackend, Statement}; pub struct NewBlocksStatement; impl StatementFromRange for NewBlocksStatement { - fn get_statement(range: Option>, _: &BlockscoutMigrations) -> Statement { + fn get_statement(range: Option>>, _: &BlockscoutMigrations) -> Statement { sql_with_range_filter_opt!( DbBackend::Postgres, r#" diff --git a/stats/stats/src/charts/lines/new_contracts.rs b/stats/stats/src/charts/lines/new_contracts.rs index 958d3893f..0c7ec3d51 100644 --- a/stats/stats/src/charts/lines/new_contracts.rs +++ b/stats/stats/src/charts/lines/new_contracts.rs @@ -25,14 +25,15 @@ use crate::{ }; use chrono::NaiveDate; +use chrono::{DateTime, Utc}; use entity::sea_orm_active_enums::ChartType; -use sea_orm::{prelude::DateTimeUtc, DbBackend, Statement}; +use sea_orm::{DbBackend, Statement}; pub struct NewContractsStatement; impl StatementFromRange for NewContractsStatement { fn get_statement( - range: Option>, + range: Option>>, completed_migrations: &BlockscoutMigrations, ) -> Statement { if completed_migrations.denormalization { diff --git a/stats/stats/src/charts/lines/new_native_coin_transfers.rs b/stats/stats/src/charts/lines/new_native_coin_transfers.rs index 2120256a5..db9874def 100644 --- a/stats/stats/src/charts/lines/new_native_coin_transfers.rs +++ b/stats/stats/src/charts/lines/new_native_coin_transfers.rs @@ -25,14 +25,15 @@ use crate::{ }; use chrono::NaiveDate; +use chrono::{DateTime, Utc}; use entity::sea_orm_active_enums::ChartType; -use sea_orm::{prelude::*, DbBackend, Statement}; +use sea_orm::{DbBackend, Statement}; pub struct NewNativeCoinTransfersStatement; impl StatementFromRange for NewNativeCoinTransfersStatement { fn get_statement( - range: Option>, + range: Option>>, completed_migrations: &BlockscoutMigrations, ) -> Statement { if completed_migrations.denormalization { diff --git a/stats/stats/src/charts/lines/new_txns.rs b/stats/stats/src/charts/lines/new_txns.rs index e4ef6a5c5..6fec2a7f6 100644 --- a/stats/stats/src/charts/lines/new_txns.rs +++ b/stats/stats/src/charts/lines/new_txns.rs @@ -25,14 +25,15 @@ use crate::{ }; use chrono::NaiveDate; +use chrono::{DateTime, Utc}; use entity::sea_orm_active_enums::ChartType; -use sea_orm::{prelude::*, DbBackend, Statement}; +use sea_orm::{DbBackend, Statement}; pub struct NewTxnsStatement; impl StatementFromRange for NewTxnsStatement { fn get_statement( - range: Option>, + range: Option>>, completed_migrations: &BlockscoutMigrations, ) -> Statement { if completed_migrations.denormalization { diff --git a/stats/stats/src/charts/lines/new_verified_contracts.rs b/stats/stats/src/charts/lines/new_verified_contracts.rs index 2778587a8..a677918d4 100644 --- a/stats/stats/src/charts/lines/new_verified_contracts.rs +++ b/stats/stats/src/charts/lines/new_verified_contracts.rs @@ -25,13 +25,14 @@ use crate::{ }; use chrono::NaiveDate; +use chrono::{DateTime, Utc}; use entity::sea_orm_active_enums::ChartType; -use sea_orm::{prelude::*, DbBackend, Statement}; +use sea_orm::{DbBackend, Statement}; pub struct NewVerifiedContractsStatement; impl StatementFromRange for NewVerifiedContractsStatement { - fn get_statement(range: Option>, _: &BlockscoutMigrations) -> Statement { + fn get_statement(range: Option>>, _: &BlockscoutMigrations) -> Statement { sql_with_range_filter_opt!( DbBackend::Postgres, r#" diff --git a/stats/stats/src/charts/lines/txns_fee.rs b/stats/stats/src/charts/lines/txns_fee.rs index ad4326591..326ddd99d 100644 --- a/stats/stats/src/charts/lines/txns_fee.rs +++ b/stats/stats/src/charts/lines/txns_fee.rs @@ -27,8 +27,9 @@ use crate::{ }; use chrono::NaiveDate; +use chrono::{DateTime, Utc}; use entity::sea_orm_active_enums::ChartType; -use sea_orm::{prelude::*, DbBackend, Statement}; +use sea_orm::{DbBackend, Statement}; const ETHER: i64 = i64::pow(10, 18); @@ -36,7 +37,7 @@ pub struct TxnsFeeStatement; impl StatementFromRange for TxnsFeeStatement { fn get_statement( - range: Option>, + range: Option>>, completed_migrations: &BlockscoutMigrations, ) -> Statement { if completed_migrations.denormalization { diff --git a/stats/stats/src/charts/lines/txns_success_rate.rs b/stats/stats/src/charts/lines/txns_success_rate.rs index 1df1d3370..bc122dd27 100644 --- a/stats/stats/src/charts/lines/txns_success_rate.rs +++ b/stats/stats/src/charts/lines/txns_success_rate.rs @@ -25,8 +25,9 @@ use crate::{ }; use chrono::NaiveDate; +use chrono::{DateTime, Utc}; use entity::sea_orm_active_enums::ChartType; -use sea_orm::{prelude::*, DbBackend, Statement}; +use sea_orm::{DbBackend, Statement}; use super::new_txns::{NewTxnsInt, NewTxnsMonthlyInt}; @@ -34,7 +35,7 @@ pub struct TxnsSuccessRateStatement; impl StatementFromRange for TxnsSuccessRateStatement { fn get_statement( - range: Option>, + range: Option>>, completed_migrations: &BlockscoutMigrations, ) -> Statement { if completed_migrations.denormalization { diff --git a/stats/stats/src/data_source/tests.rs b/stats/stats/src/data_source/tests.rs index 887646a50..f1eb97fd8 100644 --- a/stats/stats/src/data_source/tests.rs +++ b/stats/stats/src/data_source/tests.rs @@ -2,7 +2,7 @@ use std::{collections::HashSet, ops::Range, str::FromStr, sync::Arc}; use chrono::{DateTime, NaiveDate, Utc}; use entity::sea_orm_active_enums::ChartType; -use sea_orm::{prelude::*, DbBackend, Statement}; +use sea_orm::{DatabaseConnection, DbBackend, Statement}; use tokio::sync::Mutex; use super::{ @@ -45,7 +45,7 @@ pub struct NewContractsQuery; impl StatementFromRange for NewContractsQuery { fn get_statement( - range: Option>, + range: Option>>, completed_migrations: &BlockscoutMigrations, ) -> Statement { // choose the statement based on migration progress diff --git a/stats/stats/src/update_group.rs b/stats/stats/src/update_group.rs index 076f36098..f8a2a800f 100644 --- a/stats/stats/src/update_group.rs +++ b/stats/stats/src/update_group.rs @@ -130,7 +130,6 @@ pub trait UpdateGroup: core::fmt::Debug { /// # use chrono::NaiveDate; /// # use entity::sea_orm_active_enums::ChartType; /// # use std::ops::RangeInclusive; -/// # use sea_orm::prelude::DateTimeUtc; /// # use sea_orm::Statement; /// # /// # struct DummyChartProperties; @@ -211,13 +210,12 @@ pub trait UpdateGroup: core::fmt::Debug { /// # use chrono::NaiveDate; /// # use entity::sea_orm_active_enums::ChartType; /// # use std::ops::Range; -/// # use sea_orm::prelude::DateTimeUtc; /// # use sea_orm::Statement; /// /// struct DummyRemoteStatement; /// /// impl StatementFromRange for DummyRemoteStatement { -/// fn get_statement(range: Option>, _: &BlockscoutMigrations) -> Statement { +/// fn get_statement(range: Option>>, _: &BlockscoutMigrations) -> Statement { /// todo!() /// } /// } diff --git a/stats/stats/src/utils.rs b/stats/stats/src/utils.rs index 54a6c1f59..75deaf3bb 100644 --- a/stats/stats/src/utils.rs +++ b/stats/stats/src/utils.rs @@ -94,8 +94,8 @@ mod test { ("".to_string(), vec![]) ); - let time_1 = DateTimeUtc::from_timestamp(1234567, 0).unwrap(); - let time_2 = DateTimeUtc::from_timestamp(7654321, 0).unwrap(); + let time_1 = DateTime::::from_timestamp(1234567, 0).unwrap(); + let time_2 = DateTime::::from_timestamp(7654321, 0).unwrap(); assert_eq!( produce_filter_and_values(Some(time_1..time_2), "aboba", 123), ( @@ -110,7 +110,7 @@ mod test { const ETH: i64 = 1_000_000_000_000_000_000; - fn naive_sql_selector(range: Option>) -> Statement { + fn naive_sql_selector(range: Option>>) -> Statement { match range { Some(range) => Statement::from_sql_and_values( DbBackend::Postgres, @@ -175,8 +175,8 @@ mod test { #[test] fn sql_with_range_filter_some_works() { let range = Some( - DateTimeUtc::from_timestamp(1234567, 0).unwrap() - ..DateTimeUtc::from_timestamp(7654321, 0).unwrap(), + DateTime::::from_timestamp(1234567, 0).unwrap() + ..DateTime::::from_timestamp(7654321, 0).unwrap(), ); assert_eq!( compact_sql(naive_sql_selector(range.clone())), From cdb8af24c0c36098fdb71dec2a7f2e70b3ce7431 Mon Sep 17 00:00:00 2001 From: Kirill Ivanov Date: Mon, 2 Dec 2024 22:35:08 +0900 Subject: [PATCH 15/41] fix build lol --- stats/stats/src/lib.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/stats/stats/src/lib.rs b/stats/stats/src/lib.rs index 546af418f..44efdc4eb 100644 --- a/stats/stats/src/lib.rs +++ b/stats/stats/src/lib.rs @@ -1,3 +1,5 @@ +#![recursion_limit = "256"] + mod charts; pub mod data_processing; pub mod data_source; From 68abc79a37c58dc857de945215481b467e903822 Mon Sep 17 00:00:00 2001 From: Kirill Ivanov Date: Mon, 2 Dec 2024 22:37:10 +0900 Subject: [PATCH 16/41] clippy --- stats/stats-server/src/read_service.rs | 2 +- stats/stats/src/charts/db_interaction/read.rs | 2 +- stats/stats/src/charts/query_dispatch.rs | 12 ++++++------ stats/stats/src/range.rs | 2 +- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/stats/stats-server/src/read_service.rs b/stats/stats-server/src/read_service.rs index 219c93dfe..7d919d024 100644 --- a/stats/stats-server/src/read_service.rs +++ b/stats/stats-server/src/read_service.rs @@ -186,7 +186,7 @@ impl StatsService for ReadService { name.clone(), counter.settings.clone(), query_handle, - now.clone(), + now, )) }) .collect(); diff --git a/stats/stats/src/charts/db_interaction/read.rs b/stats/stats/src/charts/db_interaction/read.rs index 8e09f3c43..abb442004 100644 --- a/stats/stats/src/charts/db_interaction/read.rs +++ b/stats/stats/src/charts/db_interaction/read.rs @@ -504,7 +504,7 @@ pub async fn get_estimated_table_rows( let count = CountEstimate::find_by_statement(statement) .one(blockscout) .await?; - let count = count.map(|c| c.count).flatten(); + let count = count.and_then(|c| c.count); Ok(count) } diff --git a/stats/stats/src/charts/query_dispatch.rs b/stats/stats/src/charts/query_dispatch.rs index 8b79f409a..83f855759 100644 --- a/stats/stats/src/charts/query_dispatch.rs +++ b/stats/stats/src/charts/query_dispatch.rs @@ -80,15 +80,15 @@ impl ChartTypeSpecifics { } } -impl Into for CounterHandle { - fn into(self) -> ChartTypeSpecifics { - ChartTypeSpecifics::Counter { query: self } +impl From for ChartTypeSpecifics { + fn from(val: CounterHandle) -> Self { + ChartTypeSpecifics::Counter { query: val } } } -impl Into for LineHandle { - fn into(self) -> ChartTypeSpecifics { - ChartTypeSpecifics::Line { query: self } +impl From for ChartTypeSpecifics { + fn from(val: LineHandle) -> Self { + ChartTypeSpecifics::Line { query: val } } } diff --git a/stats/stats/src/range.rs b/stats/stats/src/range.rs index 904c7d260..9efb733ed 100644 --- a/stats/stats/src/range.rs +++ b/stats/stats/src/range.rs @@ -78,7 +78,7 @@ impl From>> for UniversalRange { fn from(value: RangeInclusive>) -> Self { let (start, end) = value.into_inner(); Self { - start: start, + start, end: match end { Some(e) => Bound::Included(e), None => Bound::Unbounded, From b655bc46e13de25072b35c507ef683739d42e119 Mon Sep 17 00:00:00 2001 From: Kirill Ivanov Date: Tue, 3 Dec 2024 13:43:50 +0900 Subject: [PATCH 17/41] remove raw counters fetch --- .../src/charts/counters/average_block_time.rs | 4 +- .../stats/src/charts/counters/total_blocks.rs | 36 +++++++---- stats/stats/src/charts/db_interaction/read.rs | 63 +++++-------------- stats/stats/src/charts/query_dispatch.rs | 24 +++++-- .../src/data_source/kinds/local_db/mod.rs | 6 -- stats/stats/src/lib.rs | 4 +- stats/stats/src/tests/simple_test.rs | 40 +++++++----- stats/stats/src/update_group.rs | 28 +++++++-- 8 files changed, 110 insertions(+), 95 deletions(-) diff --git a/stats/stats/src/charts/counters/average_block_time.rs b/stats/stats/src/charts/counters/average_block_time.rs index 761b34346..4c51295e6 100644 --- a/stats/stats/src/charts/counters/average_block_time.rs +++ b/stats/stats/src/charts/counters/average_block_time.rs @@ -207,14 +207,14 @@ mod tests { AverageBlockTime::update_recursively(&cx).await.unwrap(); assert_eq!( expected_avg.to_string(), - get_counter::(&db).await + get_counter::(&cx).await.value ); parameters.force_full = false; let cx = UpdateContext::from_params_now_or_override(parameters.clone()); AverageBlockTime::update_recursively(&cx).await.unwrap(); assert_eq!( expected_avg.to_string(), - get_counter::(&db).await + get_counter::(&cx).await.value ); } diff --git a/stats/stats/src/charts/counters/total_blocks.rs b/stats/stats/src/charts/counters/total_blocks.rs index f20820b7d..77c784e19 100644 --- a/stats/stats/src/charts/counters/total_blocks.rs +++ b/stats/stats/src/charts/counters/total_blocks.rs @@ -110,14 +110,17 @@ mod tests { use super::*; use crate::{ data_source::{types::BlockscoutMigrations, DataSource, UpdateContext, UpdateParameters}, - get_raw_counters, - tests::{init_db::init_db_all, mock_blockscout::fill_mock_blockscout_data}, - ChartObject, Named, + query_dispatch::QuerySerialized, + tests::{ + init_db::init_db_all, mock_blockscout::fill_mock_blockscout_data, + simple_test::get_counter, + }, + ChartObject, }; use chrono::NaiveDate; use entity::chart_data; - use pretty_assertions::assert_eq; - use sea_orm::{DatabaseConnection, EntityTrait, Set}; + use pretty_assertions::{assert_eq, assert_ne}; + use sea_orm::{DatabaseConnection, DbBackend, EntityTrait, Set, Statement}; use std::str::FromStr; #[tokio::test] @@ -153,8 +156,8 @@ mod tests { }; let cx = UpdateContext::from_params_now_or_override(parameters.clone()); TotalBlocks::update_recursively(&cx).await.unwrap(); - let data = get_raw_counters(&db).await.unwrap(); - assert_eq!("13", data[&TotalBlocks::name()].value); + let data = get_counter::(&cx).await; + assert_eq!("13", data.value); } #[tokio::test] @@ -180,8 +183,8 @@ mod tests { }; let cx = UpdateContext::from_params_now_or_override(parameters.clone()); TotalBlocks::update_recursively(&cx).await.unwrap(); - let data = get_raw_counters(&db).await.unwrap(); - assert_eq!("9", data[&TotalBlocks::name()].value); + let data = get_counter::(&cx).await; + assert_eq!("9", data.value); } #[tokio::test] @@ -217,8 +220,8 @@ mod tests { }; let cx = UpdateContext::from_params_now_or_override(parameters.clone()); TotalBlocks::update_recursively(&cx).await.unwrap(); - let data = get_raw_counters(&db).await.unwrap(); - assert_eq!("13", data[&TotalBlocks::name()].value); + let data = get_counter::(&cx).await; + assert_eq!("13", data.value); } #[tokio::test] @@ -238,6 +241,13 @@ mod tests { fill_mock_blockscout_data(&blockscout, current_date).await; + // need to analyze or vacuum for `reltuples` to be updated. + // source: https://www.postgresql.org/docs/9.3/planner-stats.html + let _ = blockscout + .execute(Statement::from_string(DbBackend::Postgres, "ANALYZE;")) + .await + .unwrap(); + let parameters = UpdateParameters { db: &db, blockscout: &blockscout, @@ -246,7 +256,7 @@ mod tests { force_full: false, }; let cx: UpdateContext<'_> = UpdateContext::from_params_now_or_override(parameters.clone()); - let data = get_raw_counters(&db).await.unwrap(); - // assert_eq!("13", data[&TotalBlocks::name()].value); + let data = get_counter::(&cx).await; + assert_ne!("0", data.value); } } diff --git a/stats/stats/src/charts/db_interaction/read.rs b/stats/stats/src/charts/db_interaction/read.rs index abb442004..e5acdabf5 100644 --- a/stats/stats/src/charts/db_interaction/read.rs +++ b/stats/stats/src/charts/db_interaction/read.rs @@ -22,7 +22,7 @@ use sea_orm::{ ColumnTrait, ConnectionTrait, DatabaseConnection, DbBackend, DbErr, EntityTrait, FromQueryResult, QueryFilter, QueryOrder, QuerySelect, Statement, }; -use std::{collections::HashMap, fmt::Debug, ops::Range}; +use std::{fmt::Debug, ops::Range}; use thiserror::Error; use tracing::instrument; @@ -55,48 +55,6 @@ pub async fn find_chart(db: &DatabaseConnection, chart: &ChartKey) -> Result Result>, ReadError> { - let data = CounterData::find_by_statement(Statement::from_string( - DbBackend::Postgres, - r#" - SELECT distinct on (charts.id) charts.name, data.date, data.value - FROM "chart_data" "data" - INNER JOIN "charts" - ON data.chart_id = charts.id - WHERE charts.chart_type = 'COUNTER' - ORDER BY charts.id, data.id DESC; - "#, - )) - .all(db) - .await?; - - let counters: HashMap<_, _> = data - .into_iter() - .map(|data| { - ( - data.name, - DateValue:: { - timespan: data.date, - value: data.value, - }, - ) - }) - .collect(); - - Ok(counters) -} - /// Get counter value for the requested date pub async fn get_counter_data( db: &DatabaseConnection, @@ -713,12 +671,16 @@ mod tests { use crate::{ charts::ResolutionKind, counters::TotalBlocks, - data_source::kinds::local_db::parameters::DefaultQueryVec, + data_source::{ + kinds::local_db::parameters::DefaultQueryVec, types::BlockscoutMigrations, + UpdateParameters, + }, lines::{AccountsGrowth, ActiveAccounts, TxnsGrowth, TxnsGrowthMonthly}, tests::{ init_db::{init_db, init_db_all}, mock_blockscout::fill_mock_blockscout_data, point_construction::{d, month_of}, + simple_test::get_counter, }, types::timespans::Month, Named, @@ -868,10 +830,17 @@ mod tests { let db = init_db("get_counters_mock").await; insert_mock_data(&db).await; - let counters = get_raw_counters(&db).await.unwrap(); + let cx = UpdateContext::from_params_now_or_override(UpdateParameters { + db: &db, + // shouldn't use this because mock data contains total blocks value + blockscout: &db, + blockscout_applied_migrations: BlockscoutMigrations::latest(), + update_time_override: None, + force_full: false, + }); assert_eq!( - HashMap::from_iter([("totalBlocks".into(), value("2022-11-12", "1350"))]), - counters + value("2022-11-12", "1350"), + get_counter::(&cx).await ); } diff --git a/stats/stats/src/charts/query_dispatch.rs b/stats/stats/src/charts/query_dispatch.rs index 83f855759..169b67f6f 100644 --- a/stats/stats/src/charts/query_dispatch.rs +++ b/stats/stats/src/charts/query_dispatch.rs @@ -26,6 +26,12 @@ pub trait QuerySerialized { /// Currently `Point` or `Vec` type Output: Send; + /// `new` function that is created solely for the purposes of + /// dynamic dispatch (see where it's used). + fn new_for_dynamic_dispatch() -> Self + where + Self: Sized; + /// Retrieve chart data from local storage. fn query_data<'a>( &self, @@ -118,14 +124,15 @@ impl SerializableQueryOutput for TimespanValue { impl QuerySerialized for LocalDbChartSource where - MainDep: DataSource, - ResolutionDep: DataSource, - Create: CreateBehaviour, - Update: UpdateBehaviour, - Query: QueryBehaviour, + MainDep: DataSource + Sync, + ResolutionDep: DataSource + Sync, + Create: CreateBehaviour + Sync, + Update: UpdateBehaviour + Sync, + Query: QueryBehaviour + Sync, QueryOutput: SerializableQueryOutput, QueryOutput::Serialized: Send, ChartProps: ChartProperties, + ChartProps::Resolution: Ord + Clone + Debug, { type Output = QueryOutput::Serialized; @@ -143,6 +150,13 @@ where Ok(data.serialize()) }) } + + fn new_for_dynamic_dispatch() -> Self + where + Self: Sized, + { + Self(std::marker::PhantomData) + } } fn serialize_point( diff --git a/stats/stats/src/data_source/kinds/local_db/mod.rs b/stats/stats/src/data_source/kinds/local_db/mod.rs index b38843df7..f45a82186 100644 --- a/stats/stats/src/data_source/kinds/local_db/mod.rs +++ b/stats/stats/src/data_source/kinds/local_db/mod.rs @@ -136,12 +136,6 @@ where ChartProps: ChartProperties, ChartProps::Resolution: Ord + Clone + Debug, { - /// `new` function that is created solely for the purposes of - /// dynamic dispatch (see where it's used). - pub fn new_for_dynamic_dispatch() -> Self { - Self(PhantomData) - } - /// Performs common checks and prepares values useful for further /// update. Then proceeds to update according to parameters. async fn update_itself_inner( diff --git a/stats/stats/src/lib.rs b/stats/stats/src/lib.rs index 44efdc4eb..fa477611b 100644 --- a/stats/stats/src/lib.rs +++ b/stats/stats/src/lib.rs @@ -19,8 +19,8 @@ pub use migration; pub use charts::{ counters, db_interaction::read::{ - get_line_chart_data, get_raw_counters, ApproxUnsignedDiff, QueryAllBlockTimestampRange, - ReadError, RequestedPointsLimit, + get_line_chart_data, ApproxUnsignedDiff, QueryAllBlockTimestampRange, ReadError, + RequestedPointsLimit, }, lines, query_dispatch, types, ChartKey, ChartObject, ChartProperties, ChartPropertiesObject, MissingDatePolicy, Named, ResolutionKind, UpdateError, diff --git a/stats/stats/src/tests/simple_test.rs b/stats/stats/src/tests/simple_test.rs index cdd4a0fd0..c9e872fb1 100644 --- a/stats/stats/src/tests/simple_test.rs +++ b/stats/stats/src/tests/simple_test.rs @@ -4,8 +4,10 @@ use crate::{ source::DataSource, types::{BlockscoutMigrations, UpdateContext, UpdateParameters}, }, - get_line_chart_data, get_raw_counters, - types::Timespan, + get_line_chart_data, + query_dispatch::QuerySerialized, + range::UniversalRange, + types::{timespans::DateValue, Timespan}, ChartProperties, MissingDatePolicy, }; use blockscout_service_launcher::test_database::TestDbGuard; @@ -306,11 +308,13 @@ where } /// `test_name` must be unique to avoid db clashes -pub async fn simple_test_counter( +pub async fn simple_test_counter( test_name: &str, expected: &str, update_time: Option, -) { +) where + C: DataSource + ChartProperties + QuerySerialized>, +{ simple_test_counter_inner::( test_name, expected, @@ -327,23 +331,27 @@ pub async fn simple_test_counter( /// - db is going to be initialized separately for each variant /// - `_N` will be added to `test_name_base` for each variant /// - the resulting test name must be unique to avoid db clashes -pub async fn simple_test_counter_with_migration_variants( +pub async fn simple_test_counter_with_migration_variants( test_name_base: &str, expected: &str, update_time: Option, -) { +) where + C: DataSource + ChartProperties + QuerySerialized>, +{ for (i, migrations) in MIGRATIONS_VARIANTS.into_iter().enumerate() { let test_name = format!("{test_name_base}_{i}"); simple_test_counter_inner::(&test_name, expected, update_time, migrations).await } } -async fn simple_test_counter_inner( +async fn simple_test_counter_inner( test_name: &str, expected: &str, update_time: Option, migrations: BlockscoutMigrations, -) { +) where + C: DataSource + ChartProperties + QuerySerialized>, +{ let (current_time, db, blockscout) = prepare_chart_test::(test_name, update_time).await; let max_time = DateTime::::from_str("2023-03-01T12:00:00Z").unwrap(); let max_date = max_time.date_naive(); @@ -358,11 +366,11 @@ async fn simple_test_counter_inner( }; let cx = UpdateContext::from_params_now_or_override(parameters.clone()); C::update_recursively(&cx).await.unwrap(); - assert_eq!(expected, get_counter::(&db).await); + assert_eq!(expected, get_counter::(&cx).await.value); parameters.force_full = false; let cx = UpdateContext::from_params_now_or_override(parameters.clone()); C::update_recursively(&cx).await.unwrap(); - assert_eq!(expected, get_counter::(&db).await); + assert_eq!(expected, get_counter::(&cx).await.value); } pub async fn prepare_chart_test( @@ -378,8 +386,12 @@ pub async fn prepare_chart_test( (init_time, db, blockscout) } -pub async fn get_counter(db: &DatabaseConnection) -> String { - let data = get_raw_counters(db).await.unwrap(); - let data = &data[&C::name()]; - data.value.clone() +pub async fn get_counter>>( + cx: &UpdateContext<'_>, +) -> DateValue { + let counter_object = ::new_for_dynamic_dispatch(); + counter_object + .query_data(cx, UniversalRange::full(), None, false) + .await + .unwrap() } diff --git a/stats/stats/src/update_group.rs b/stats/stats/src/update_group.rs index f8a2a800f..59c187ba3 100644 --- a/stats/stats/src/update_group.rs +++ b/stats/stats/src/update_group.rs @@ -264,7 +264,9 @@ macro_rules! construct_update_group { fn list_charts(&self) -> ::std::vec::Vec<$crate::ChartObject> { std::vec![ $( - $crate::ChartObject::construct_from_chart::<$member>(<$member>::new_for_dynamic_dispatch()), + $crate::ChartObject::construct_from_chart::<$member>( + <$member as $crate::query_dispatch::QuerySerialized>::new_for_dynamic_dispatch() + ), )* ] } @@ -272,15 +274,23 @@ macro_rules! construct_update_group { fn list_dependency_mutex_ids(&self) -> ::std::collections::HashSet { let mut ids = ::std::collections::HashSet::new(); $( - ids.extend(<$member as $crate::data_source::DataSource>::all_dependencies_mutex_ids().into_iter()); + ids.extend( + <$member as $crate::data_source::DataSource>::all_dependencies_mutex_ids() + .into_iter() + ); )* ids } - fn dependency_mutex_ids_of(&self, chart_id: &$crate::ChartKey) -> Option<::std::collections::HashSet> { + fn dependency_mutex_ids_of( + &self, + chart_id: &$crate::ChartKey + ) -> Option<::std::collections::HashSet> { $( if chart_id == &<$member as $crate::ChartProperties>::key() { - return Some(<$member as $crate::data_source::DataSource>::all_dependencies_mutex_ids()); + return Some( + <$member as $crate::data_source::DataSource>::all_dependencies_mutex_ids() + ); } )* return None; @@ -300,7 +310,9 @@ macro_rules! construct_update_group { let current_time = creation_time_override.unwrap_or_else(|| ::chrono::Utc::now()); $( if enabled_charts.contains(&<$member as $crate::ChartProperties>::key()) { - <$member as $crate::data_source::DataSource>::init_recursively(db, ¤t_time).await?; + <$member as $crate::data_source::DataSource>::init_recursively( + db, ¤t_time + ).await?; } )* Ok(()) @@ -308,7 +320,11 @@ macro_rules! construct_update_group { // updates are expected to be unique by group name & update time; this instrumentation // should allow to single out one update process in logs - #[::tracing::instrument(skip_all, fields(update_group=self.name(), update_time), level = tracing::Level::INFO)] + #[::tracing::instrument( + skip_all, + fields(update_group=self.name(), update_time), + level = tracing::Level::INFO + )] async fn update_charts<'a>( &self, params: $crate::data_source::UpdateParameters<'a>, From 4a4a95900a366131365a34ced352c82c2b00b4ae Mon Sep 17 00:00:00 2001 From: Kirill Ivanov Date: Wed, 4 Dec 2024 17:00:24 +0900 Subject: [PATCH 18/41] mark db connections to avoid cache clashing --- stats/stats-server/src/read_service.rs | 14 ++--- stats/stats-server/src/server.rs | 12 +++-- stats/stats-server/src/update_service.rs | 26 ++++++---- .../src/charts/counters/average_block_time.rs | 7 +-- .../stats/src/charts/counters/total_blocks.rs | 52 ++++++++++--------- .../src/charts/counters/total_contracts.rs | 3 +- .../src/charts/counters/yesterday_txns.rs | 3 +- stats/stats/src/charts/db_interaction/read.rs | 7 +-- .../lines/native_coin_holders_growth.rs | 40 ++++++++------ stats/stats/src/charts/lines/new_accounts.rs | 2 +- .../src/charts/lines/new_block_rewards.rs | 8 +-- stats/stats/src/charts/lines/new_blocks.rs | 46 ++++++++-------- .../data_manipulation/filter_deducible.rs | 7 ++- .../data_manipulation/resolutions/average.rs | 19 +++++-- .../resolutions/last_value.rs | 7 ++- .../data_manipulation/resolutions/sum.rs | 7 ++- .../src/data_source/kinds/local_db/mod.rs | 19 +++---- .../kinds/local_db/parameters/query.rs | 24 +++++---- .../parameters/update/batching/mod.rs | 11 ++-- .../kinds/local_db/parameters/update/point.rs | 2 +- .../data_source/kinds/remote_db/query/all.rs | 2 +- .../data_source/kinds/remote_db/query/each.rs | 2 +- .../data_source/kinds/remote_db/query/one.rs | 2 +- stats/stats/src/data_source/tests.rs | 8 +-- stats/stats/src/data_source/types.rs | 10 ++-- stats/stats/src/tests/init_db.rs | 10 ++++ stats/stats/src/tests/simple_test.rs | 34 +++++++----- stats/stats/src/utils.rs | 32 +++++++++++- 28 files changed, 261 insertions(+), 155 deletions(-) diff --git a/stats/stats-server/src/read_service.rs b/stats/stats-server/src/read_service.rs index 7d919d024..c250b42d6 100644 --- a/stats/stats-server/src/read_service.rs +++ b/stats/stats-server/src/read_service.rs @@ -13,13 +13,13 @@ use async_trait::async_trait; use chrono::{DateTime, NaiveDate, Utc}; use futures::{stream::FuturesOrdered, StreamExt}; use proto_v1::stats_service_server::StatsService; -use sea_orm::{DatabaseConnection, DbErr}; +use sea_orm::DbErr; use stats::{ data_source::{types::BlockscoutMigrations, UpdateContext, UpdateParameters}, query_dispatch::{CounterHandle, LineHandle, QuerySerializedDyn}, range::UniversalRange, types::Timespan, - utils::day_start, + utils::{day_start, MarkedDbConnection}, RequestedPointsLimit, ResolutionKind, UpdateError, }; use stats_proto::blockscout::stats::v1 as proto_v1; @@ -27,16 +27,16 @@ use tonic::{Request, Response, Status}; #[derive(Clone)] pub struct ReadService { - db: Arc, - blockscout: Arc, + db: MarkedDbConnection, + blockscout: MarkedDbConnection, charts: Arc, limits: ReadLimits, } impl ReadService { pub async fn new( - db: Arc, - blockscout: Arc, + db: MarkedDbConnection, + blockscout: MarkedDbConnection, charts: Arc, limits: ReadLimits, ) -> Result { @@ -104,7 +104,7 @@ impl ReadService { points_limit: Option, query_time: DateTime, ) -> Result { - let migrations = BlockscoutMigrations::query_from_db(&self.blockscout) + let migrations = BlockscoutMigrations::query_from_db(self.blockscout.connection.as_ref()) .await .map_err(UpdateError::BlockscoutDB)?; let context = UpdateContext::from_params_now_or_override(UpdateParameters { diff --git a/stats/stats-server/src/server.rs b/stats/stats-server/src/server.rs index f0a1da110..16f62db3d 100644 --- a/stats/stats-server/src/server.rs +++ b/stats/stats-server/src/server.rs @@ -14,7 +14,7 @@ use anyhow::Context; use blockscout_endpoint_swagger::route_swagger; use blockscout_service_launcher::launcher::{self, LaunchSettings}; use sea_orm::{ConnectOptions, Database}; -use stats::metrics; +use stats::{metrics, utils::MarkedDbConnection}; use stats_proto::blockscout::stats::v1::{ health_actix::route_health, health_server::HealthServer, @@ -79,7 +79,9 @@ pub async fn stats(mut settings: Settings) -> Result<(), anyhow::Error> { settings.run_migrations, ) .await?; - let db = Arc::new(Database::connect(opt).await.context("stats DB")?); + let db = MarkedDbConnection::main_connection(Arc::new( + Database::connect(opt).await.context("stats DB")?, + )); let mut opt = ConnectOptions::new(settings.blockscout_db_url.clone()); opt.sqlx_logging_level(tracing::log::LevelFilter::Debug); @@ -89,7 +91,9 @@ pub async fn stats(mut settings: Settings) -> Result<(), anyhow::Error> { tracing::log::LevelFilter::Warn, Duration::from_secs(3600), ); - let blockscout = Arc::new(Database::connect(opt).await.context("blockscout DB")?); + let blockscout = MarkedDbConnection::main_connection(Arc::new( + Database::connect(opt).await.context("blockscout DB")?, + )); let charts = Arc::new(RuntimeSetup::new( charts_config, @@ -101,7 +105,7 @@ pub async fn stats(mut settings: Settings) -> Result<(), anyhow::Error> { for group_entry in charts.update_groups.values() { group_entry .group - .create_charts_with_mutexes(&db, None, &group_entry.enabled_members) + .create_charts_with_mutexes(db.connection.as_ref(), None, &group_entry.enabled_members) .await?; } diff --git a/stats/stats-server/src/update_service.rs b/stats/stats-server/src/update_service.rs index 8c05f9a80..543d820f7 100644 --- a/stats/stats-server/src/update_service.rs +++ b/stats/stats-server/src/update_service.rs @@ -1,16 +1,19 @@ use crate::runtime_setup::{RuntimeSetup, UpdateGroupEntry}; use chrono::Utc; use cron::Schedule; -use sea_orm::{DatabaseConnection, DbErr}; -use stats::data_source::types::{BlockscoutMigrations, UpdateParameters}; +use sea_orm::DbErr; +use stats::{ + data_source::types::{BlockscoutMigrations, UpdateParameters}, + utils::MarkedDbConnection, +}; use std::sync::Arc; use tokio::task::JoinHandle; const FAILED_UPDATERS_UNTIL_PANIC: u64 = 3; pub struct UpdateService { - db: Arc, - blockscout: Arc, + db: MarkedDbConnection, + blockscout: MarkedDbConnection, charts: Arc, } @@ -26,8 +29,8 @@ fn time_till_next_call(schedule: &Schedule) -> std::time::Duration { impl UpdateService { pub async fn new( - db: Arc, - blockscout: Arc, + db: MarkedDbConnection, + blockscout: MarkedDbConnection, charts: Arc, ) -> Result { Ok(Self { @@ -104,11 +107,12 @@ impl UpdateService { force_update = force_full, "updating group of charts" ); - let Ok(active_migrations) = BlockscoutMigrations::query_from_db(&self.blockscout) - .await - .inspect_err(|err| { - tracing::error!("error during blockscout migrations detection: {:?}", err) - }) + let Ok(active_migrations) = + BlockscoutMigrations::query_from_db(self.blockscout.connection.as_ref()) + .await + .inspect_err(|err| { + tracing::error!("error during blockscout migrations detection: {:?}", err) + }) else { return; }; diff --git a/stats/stats/src/charts/counters/average_block_time.rs b/stats/stats/src/charts/counters/average_block_time.rs index 4c51295e6..bb9b05c81 100644 --- a/stats/stats/src/charts/counters/average_block_time.rs +++ b/stats/stats/src/charts/counters/average_block_time.rs @@ -55,7 +55,7 @@ async fn query_average_block_time( ) -> Result>, UpdateError> { let query = average_block_time_statement(offset); let block_timestamps = BlockTimestamp::find_by_statement(query) - .all(cx.blockscout) + .all(cx.blockscout.connection.as_ref()) .await .map_err(UpdateError::BlockscoutDB)?; Ok(calculate_average_block_time(block_timestamps)) @@ -149,6 +149,7 @@ mod tests { mock_blockscout::fill_many_blocks, simple_test::{get_counter, prepare_chart_test, simple_test_counter}, }, + utils::MarkedDbConnection, }; #[tokio::test] @@ -197,8 +198,8 @@ mod tests { }; fill_many_blocks(&blockscout, current_time.naive_utc(), &block_times).await; let mut parameters = UpdateParameters { - db: &db, - blockscout: &blockscout, + db: &MarkedDbConnection::from_test_db(&db).unwrap(), + blockscout: &MarkedDbConnection::from_test_db(&blockscout).unwrap(), blockscout_applied_migrations: BlockscoutMigrations::latest(), update_time_override: Some(current_time), force_full: true, diff --git a/stats/stats/src/charts/counters/total_blocks.rs b/stats/stats/src/charts/counters/total_blocks.rs index 77c784e19..2adce75e0 100644 --- a/stats/stats/src/charts/counters/total_blocks.rs +++ b/stats/stats/src/charts/counters/total_blocks.rs @@ -9,6 +9,7 @@ use crate::{ }, range::UniversalRange, types::timespans::DateValue, + utils::MarkedDbConnection, ChartProperties, MissingDatePolicy, Named, UpdateError, }; @@ -38,7 +39,7 @@ impl RemoteQueryBehaviour for TotalBlocksQueryBehaviour { .column_as(Expr::col(blocks::Column::Timestamp).max(), "timestamp") .filter(blocks::Column::Consensus.eq(true)) .into_model::() - .one(cx.blockscout) + .one(cx.blockscout.connection.as_ref()) .await .map_err(UpdateError::BlockscoutDB)? .ok_or_else(|| UpdateError::Internal("query returned nothing".into()))?; @@ -75,16 +76,23 @@ impl ChartProperties for Properties { pub struct CachedBlocksEstimation; impl ValueEstimation for CachedBlocksEstimation { - async fn estimate(blockscout: &DatabaseConnection) -> Result, UpdateError> { - #[cached::proc_macro::once(time = 60, sync_writes = true, result = true)] + async fn estimate(blockscout: &MarkedDbConnection) -> Result, UpdateError> { + #[cached::proc_macro::cached( + time = 60, + key = "String", + convert = r#"{ String::from(_db_id) }"#, + sync_writes = true, + result = true + )] async fn cached_blocks_estimation( blockscout: &DatabaseConnection, + _db_id: &str, ) -> Result, DbErr> { get_estimated_table_rows(blockscout, blocks::Entity.table_name()).await } let now = Utc::now(); - let value = cached_blocks_estimation(blockscout) + let value = cached_blocks_estimation(blockscout.connection.as_ref(), &blockscout.db_name) .await .map_err(UpdateError::BlockscoutDB)? .map(|b| { @@ -110,12 +118,10 @@ mod tests { use super::*; use crate::{ data_source::{types::BlockscoutMigrations, DataSource, UpdateContext, UpdateParameters}, - query_dispatch::QuerySerialized, tests::{ - init_db::init_db_all, mock_blockscout::fill_mock_blockscout_data, + init_db::init_marked_db_all, mock_blockscout::fill_mock_blockscout_data, simple_test::get_counter, }, - ChartObject, }; use chrono::NaiveDate; use entity::chart_data; @@ -127,11 +133,11 @@ mod tests { #[ignore = "needs database to run"] async fn update_total_blocks_recurrent() { let _ = tracing_subscriber::fmt::try_init(); - let (db, blockscout) = init_db_all("update_total_blocks_recurrent").await; + let (db, blockscout) = init_marked_db_all("update_total_blocks_recurrent").await; let current_time = chrono::DateTime::from_str("2023-03-01T12:00:00Z").unwrap(); let current_date = current_time.date_naive(); - TotalBlocks::init_recursively(&db, ¤t_time) + TotalBlocks::init_recursively(&db.connection, ¤t_time) .await .unwrap(); @@ -141,11 +147,11 @@ mod tests { value: Set(1.to_string()), ..Default::default() }) - .exec(&db as &DatabaseConnection) + .exec(&db.connection as &DatabaseConnection) .await .unwrap(); - fill_mock_blockscout_data(&blockscout, current_date).await; + fill_mock_blockscout_data(&blockscout.connection, current_date).await; let parameters = UpdateParameters { db: &db, @@ -164,15 +170,15 @@ mod tests { #[ignore = "needs database to run"] async fn update_total_blocks_fresh() { let _ = tracing_subscriber::fmt::try_init(); - let (db, blockscout) = init_db_all("update_total_blocks_fresh").await; + let (db, blockscout) = init_marked_db_all("update_total_blocks_fresh").await; let current_time = chrono::DateTime::from_str("2022-11-12T12:00:00Z").unwrap(); let current_date = current_time.date_naive(); - TotalBlocks::init_recursively(&db, ¤t_time) + TotalBlocks::init_recursively(&db.connection, ¤t_time) .await .unwrap(); - fill_mock_blockscout_data(&blockscout, current_date).await; + fill_mock_blockscout_data(&blockscout.connection, current_date).await; let parameters = UpdateParameters { db: &db, @@ -191,11 +197,11 @@ mod tests { #[ignore = "needs database to run"] async fn update_total_blocks_last() { let _ = tracing_subscriber::fmt::try_init(); - let (db, blockscout) = init_db_all("update_total_blocks_last").await; + let (db, blockscout) = init_marked_db_all("update_total_blocks_last").await; let current_time = chrono::DateTime::from_str("2023-03-01T12:00:00Z").unwrap(); let current_date = current_time.date_naive(); - TotalBlocks::init_recursively(&db, ¤t_time) + TotalBlocks::init_recursively(&db.connection, ¤t_time) .await .unwrap(); @@ -205,11 +211,11 @@ mod tests { value: Set(1.to_string()), ..Default::default() }) - .exec(&db as &DatabaseConnection) + .exec(&db.connection as &DatabaseConnection) .await .unwrap(); - fill_mock_blockscout_data(&blockscout, current_date).await; + fill_mock_blockscout_data(&blockscout.connection, current_date).await; let parameters = UpdateParameters { db: &db, @@ -227,23 +233,21 @@ mod tests { #[tokio::test] #[ignore = "needs database to run"] async fn total_blocks_fallback() { - // todo: finish - ChartObject::construct_from_chart::(::new_for_dynamic_dispatch()); - let _ = tracing_subscriber::fmt::try_init(); - let (db, blockscout) = init_db_all("total_blocks_fallback").await; + let (db, blockscout) = init_marked_db_all("total_blocks_fallback").await; let current_time = chrono::DateTime::from_str("2023-03-01T12:00:00Z").unwrap(); let current_date = current_time.date_naive(); - TotalBlocks::init_recursively(&db, ¤t_time) + TotalBlocks::init_recursively(&db.connection, ¤t_time) .await .unwrap(); - fill_mock_blockscout_data(&blockscout, current_date).await; + fill_mock_blockscout_data(&blockscout.connection, current_date).await; // need to analyze or vacuum for `reltuples` to be updated. // source: https://www.postgresql.org/docs/9.3/planner-stats.html let _ = blockscout + .connection .execute(Statement::from_string(DbBackend::Postgres, "ANALYZE;")) .await .unwrap(); diff --git a/stats/stats/src/charts/counters/total_contracts.rs b/stats/stats/src/charts/counters/total_contracts.rs index a28541205..30b342fd7 100644 --- a/stats/stats/src/charts/counters/total_contracts.rs +++ b/stats/stats/src/charts/counters/total_contracts.rs @@ -1,4 +1,3 @@ - use crate::{ data_source::{ kinds::{ @@ -29,7 +28,7 @@ impl RemoteQueryBehaviour for TotalContractsQueryBehaviour { let value = addresses::Entity::find() .filter(addresses::Column::ContractCode.is_not_null()) .filter(addresses::Column::InsertedAt.lte(cx.time)) - .count(cx.blockscout) + .count(cx.blockscout.connection.as_ref()) .await .map_err(UpdateError::BlockscoutDB)?; let timespan = cx.time.date_naive(); diff --git a/stats/stats/src/charts/counters/yesterday_txns.rs b/stats/stats/src/charts/counters/yesterday_txns.rs index 929c4f72f..4cfa9a31e 100644 --- a/stats/stats/src/charts/counters/yesterday_txns.rs +++ b/stats/stats/src/charts/counters/yesterday_txns.rs @@ -1,4 +1,3 @@ - use crate::{ data_source::{ kinds::{ @@ -38,7 +37,7 @@ impl RemoteQueryBehaviour for YesterdayTxnsQuery { &cx.blockscout_applied_migrations, ); let data = Self::Output::find_by_statement(query) - .one(cx.blockscout) + .one(cx.blockscout.connection.as_ref()) .await .map_err(UpdateError::BlockscoutDB)? // no transactions for yesterday diff --git a/stats/stats/src/charts/db_interaction/read.rs b/stats/stats/src/charts/db_interaction/read.rs index e5acdabf5..af00a45bb 100644 --- a/stats/stats/src/charts/db_interaction/read.rs +++ b/stats/stats/src/charts/db_interaction/read.rs @@ -657,7 +657,7 @@ impl RemoteQueryBehaviour for QueryAllBlockTimestampRange { cx: &UpdateContext<'_>, _range: UniversalRange>, ) -> Result { - let start_timestamp = get_min_date_blockscout(cx.blockscout) + let start_timestamp = get_min_date_blockscout(cx.blockscout.connection.as_ref()) .await .map_err(UpdateError::BlockscoutDB)? .and_utc(); @@ -683,6 +683,7 @@ mod tests { simple_test::get_counter, }, types::timespans::Month, + utils::MarkedDbConnection, Named, }; use chrono::DateTime; @@ -828,8 +829,8 @@ mod tests { async fn get_counters_mock() { let _ = tracing_subscriber::fmt::try_init(); - let db = init_db("get_counters_mock").await; - insert_mock_data(&db).await; + let db = MarkedDbConnection::from_test_db(&init_db("get_counters_mock").await).unwrap(); + insert_mock_data(&db.connection).await; let cx = UpdateContext::from_params_now_or_override(UpdateParameters { db: &db, // shouldn't use this because mock data contains total blocks value diff --git a/stats/stats/src/charts/lines/native_coin_holders_growth.rs b/stats/stats/src/charts/lines/native_coin_holders_growth.rs index d1ac2e802..ef5536c89 100644 --- a/stats/stats/src/charts/lines/native_coin_holders_growth.rs +++ b/stats/stats/src/charts/lines/native_coin_holders_growth.rs @@ -118,16 +118,18 @@ pub async fn update_sequentially_with_support_table( ) -> Result<(), UpdateError> { tracing::info!(chart =% Properties::key(), "start sequential update"); let all_days = match last_accurate_point { - Some(last_row) => { - get_unique_ordered_days(cx.blockscout, Some(last_row.timespan), remote_fetch_timer) - .await - .map_err(UpdateError::BlockscoutDB)? - } + Some(last_row) => get_unique_ordered_days( + cx.blockscout.connection.as_ref(), + Some(last_row.timespan), + remote_fetch_timer, + ) + .await + .map_err(UpdateError::BlockscoutDB)?, None => { - clear_support_table(cx.db) + clear_support_table(cx.db.connection.as_ref()) .await .map_err(UpdateError::BlockscoutDB)?; - get_unique_ordered_days(cx.blockscout, None, remote_fetch_timer) + get_unique_ordered_days(cx.blockscout.connection.as_ref(), None, remote_fetch_timer) .await .map_err(UpdateError::BlockscoutDB)? } @@ -144,14 +146,22 @@ pub async fn update_sequentially_with_support_table( ); // NOTE: we update support table and chart data in one transaction // to support invariant that support table has information about last day in chart data - let db_tx = cx.db.begin().await.map_err(UpdateError::StatsDB)?; - let data: Vec = - calculate_days_using_support_table(&db_tx, cx.blockscout, days.iter().copied()) - .await - .map_err(|e| UpdateError::Internal(e.to_string()))? - .into_iter() - .map(|result| result.active_model(chart_id, Some(min_blockscout_block))) - .collect(); + let db_tx = cx + .db + .connection + .begin() + .await + .map_err(UpdateError::StatsDB)?; + let data: Vec = calculate_days_using_support_table( + &db_tx, + cx.blockscout.connection.as_ref(), + days.iter().copied(), + ) + .await + .map_err(|e| UpdateError::Internal(e.to_string()))? + .into_iter() + .map(|result| result.active_model(chart_id, Some(min_blockscout_block))) + .collect(); insert_data_many(&db_tx, data) .await .map_err(UpdateError::StatsDB)?; diff --git a/stats/stats/src/charts/lines/new_accounts.rs b/stats/stats/src/charts/lines/new_accounts.rs index 851f4b113..2c2f9f30f 100644 --- a/stats/stats/src/charts/lines/new_accounts.rs +++ b/stats/stats/src/charts/lines/new_accounts.rs @@ -110,7 +110,7 @@ impl RemoteQueryBehaviour for NewAccountsQueryBehaviour { &cx.blockscout_applied_migrations, ); let mut data = DateValue::::find_by_statement(query) - .all(cx.blockscout) + .all(cx.blockscout.connection.as_ref()) .await .map_err(UpdateError::BlockscoutDB)?; // make sure that it's sorted diff --git a/stats/stats/src/charts/lines/new_block_rewards.rs b/stats/stats/src/charts/lines/new_block_rewards.rs index 365af1aa8..b14f9f0a9 100644 --- a/stats/stats/src/charts/lines/new_block_rewards.rs +++ b/stats/stats/src/charts/lines/new_block_rewards.rs @@ -101,7 +101,7 @@ mod tests { data_source::{types::BlockscoutMigrations, DataSource, UpdateContext, UpdateParameters}, range::UniversalRange, tests::{ - init_db::init_db_all, + init_db::init_marked_db_all, mock_blockscout::fill_mock_blockscout_data, simple_test::{map_str_tuple_to_owned, simple_test_chart}, }, @@ -138,13 +138,13 @@ mod tests { ("2023-02-01", "1"), ("2023-03-01", "1"), ]); - let (db, blockscout) = init_db_all("update_new_block_rewards_monthly_int").await; + let (db, blockscout) = init_marked_db_all("update_new_block_rewards_monthly_int").await; let current_time = chrono::DateTime::from_str("2023-03-01T12:00:00Z").unwrap(); let current_date = current_time.date_naive(); - NewBlockRewardsMonthlyInt::init_recursively(&db, ¤t_time) + NewBlockRewardsMonthlyInt::init_recursively(&db.connection, ¤t_time) .await .unwrap(); - fill_mock_blockscout_data(&blockscout, current_date).await; + fill_mock_blockscout_data(&blockscout.connection, current_date).await; let parameters = UpdateParameters { db: &db, diff --git a/stats/stats/src/charts/lines/new_blocks.rs b/stats/stats/src/charts/lines/new_blocks.rs index bbef63a07..014c2d752 100644 --- a/stats/stats/src/charts/lines/new_blocks.rs +++ b/stats/stats/src/charts/lines/new_blocks.rs @@ -108,7 +108,7 @@ mod tests { data_source::{types::BlockscoutMigrations, DataSource, UpdateContext}, get_line_chart_data, tests::{ - init_db::init_db_all, mock_blockscout::fill_mock_blockscout_data, + init_db::init_marked_db_all, mock_blockscout::fill_mock_blockscout_data, point_construction::dt, simple_test::simple_test_chart, }, types::ExtendedTimespanValue, @@ -117,23 +117,25 @@ mod tests { use chrono::{NaiveDate, Utc}; use entity::{chart_data, charts}; use pretty_assertions::assert_eq; - use sea_orm::{DatabaseConnection, EntityTrait, Set}; + use sea_orm::{EntityTrait, Set}; use std::str::FromStr; #[tokio::test] #[ignore = "needs database to run"] async fn update_new_blocks_recurrent() { let _ = tracing_subscriber::fmt::try_init(); - let (db, blockscout) = init_db_all("update_new_blocks_recurrent").await; + let (db, blockscout) = init_marked_db_all("update_new_blocks_recurrent").await; let current_time = chrono::DateTime::::from_str("2022-11-12T12:00:00Z").unwrap(); let current_date = current_time.date_naive(); - fill_mock_blockscout_data(&blockscout, current_date).await; + fill_mock_blockscout_data(blockscout.connection.as_ref(), current_date).await; - NewBlocks::init_recursively(&db, ¤t_time) + NewBlocks::init_recursively(db.connection.as_ref(), ¤t_time) .await .unwrap(); - let min_blockscout_block = get_min_block_blockscout(&blockscout).await.unwrap(); + let min_blockscout_block = get_min_block_blockscout(blockscout.connection.as_ref()) + .await + .unwrap(); // set wrong value and check, that it was rewritten chart_data::Entity::insert_many([ chart_data::ActiveModel { @@ -151,7 +153,7 @@ mod tests { ..Default::default() }, ]) - .exec(&db as &DatabaseConnection) + .exec(db.connection.as_ref()) .await .unwrap(); // set corresponding `last_updated_at` for successful partial update @@ -160,7 +162,7 @@ mod tests { last_updated_at: Set(Some(dt("2022-11-12T11:00:00").and_utc().fixed_offset())), ..Default::default() }) - .exec(&db as &DatabaseConnection) + .exec(db.connection.as_ref()) .await .unwrap(); @@ -175,7 +177,7 @@ mod tests { }; NewBlocks::update_recursively(&cx).await.unwrap(); let data = get_line_chart_data::( - &db, + db.connection.as_ref(), &NewBlocks::name(), None, None, @@ -211,7 +213,7 @@ mod tests { cx.time = chrono::DateTime::::from_str("2022-11-12T13:00:00Z").unwrap(); NewBlocks::update_recursively(&cx).await.unwrap(); let data = get_line_chart_data::( - &db, + db.connection.as_ref(), &NewBlocks::name(), None, None, @@ -251,12 +253,12 @@ mod tests { #[ignore = "needs database to run"] async fn update_new_blocks_fresh() { let _ = tracing_subscriber::fmt::try_init(); - let (db, blockscout) = init_db_all("update_new_blocks_fresh").await; + let (db, blockscout) = init_marked_db_all("update_new_blocks_fresh").await; let current_time = chrono::DateTime::from_str("2022-11-12T12:00:00Z").unwrap(); let current_date = current_time.date_naive(); - fill_mock_blockscout_data(&blockscout, current_date).await; + fill_mock_blockscout_data(blockscout.connection.as_ref(), current_date).await; - NewBlocks::init_recursively(&db, ¤t_time) + NewBlocks::init_recursively(db.connection.as_ref(), ¤t_time) .await .unwrap(); @@ -269,7 +271,7 @@ mod tests { }; NewBlocks::update_recursively(&cx).await.unwrap(); let data = get_line_chart_data::( - &db, + db.connection.as_ref(), &NewBlocks::name(), None, None, @@ -309,16 +311,18 @@ mod tests { #[ignore = "needs database to run"] async fn update_new_blocks_last() { let _ = tracing_subscriber::fmt::try_init(); - let (db, blockscout) = init_db_all("update_new_blocks_last").await; + let (db, blockscout) = init_marked_db_all("update_new_blocks_last").await; let current_time = chrono::DateTime::from_str("2022-11-12T12:00:00Z").unwrap(); let current_date = current_time.date_naive(); - fill_mock_blockscout_data(&blockscout, current_date).await; + fill_mock_blockscout_data(blockscout.connection.as_ref(), current_date).await; - NewBlocks::init_recursively(&db, ¤t_time) + NewBlocks::init_recursively(db.connection.as_ref(), ¤t_time) .await .unwrap(); - let min_blockscout_block = get_min_block_blockscout(&blockscout).await.unwrap(); + let min_blockscout_block = get_min_block_blockscout(blockscout.connection.as_ref()) + .await + .unwrap(); // set wrong values and check, that they weren't rewritten // except the last one chart_data::Entity::insert_many([ @@ -351,7 +355,7 @@ mod tests { ..Default::default() }, ]) - .exec(&db as &DatabaseConnection) + .exec(db.connection.as_ref()) .await .unwrap(); // set corresponding `last_updated_at` for successful partial update @@ -360,7 +364,7 @@ mod tests { last_updated_at: Set(Some(dt("2022-11-12T11:00:00").and_utc().fixed_offset())), ..Default::default() }) - .exec(&db as &DatabaseConnection) + .exec(db.connection.as_ref()) .await .unwrap(); @@ -373,7 +377,7 @@ mod tests { }; NewBlocks::update_recursively(&cx).await.unwrap(); let data = get_line_chart_data::( - &db, + db.connection.as_ref(), &NewBlocks::name(), None, None, diff --git a/stats/stats/src/data_source/kinds/data_manipulation/filter_deducible.rs b/stats/stats/src/data_source/kinds/data_manipulation/filter_deducible.rs index 645657c30..73ce9c9ad 100644 --- a/stats/stats/src/data_source/kinds/data_manipulation/filter_deducible.rs +++ b/stats/stats/src/data_source/kinds/data_manipulation/filter_deducible.rs @@ -80,6 +80,8 @@ where #[cfg(test)] mod tests { + use std::sync::Arc; + use crate::{ data_source::types::BlockscoutMigrations, gettable_const, @@ -87,6 +89,7 @@ mod tests { range::UniversalRange, tests::point_construction::{d_v_double, dt}, types::timespans::DateValue, + utils::MarkedDbConnection, MissingDatePolicy, Named, }; @@ -153,7 +156,9 @@ mod tests { type TestedPrevious = FilterDeducible; // db is not used in mock - let empty_db = sea_orm::Database::connect("sqlite::memory:").await.unwrap(); + let empty_db = MarkedDbConnection::in_memory(Arc::new( + sea_orm::Database::connect("sqlite::memory:").await.unwrap(), + )); let context = UpdateContext { db: &empty_db, diff --git a/stats/stats/src/data_source/kinds/data_manipulation/resolutions/average.rs b/stats/stats/src/data_source/kinds/data_manipulation/resolutions/average.rs index 26c4d1dc4..0560c591e 100644 --- a/stats/stats/src/data_source/kinds/data_manipulation/resolutions/average.rs +++ b/stats/stats/src/data_source/kinds/data_manipulation/resolutions/average.rs @@ -189,7 +189,7 @@ where #[cfg(test)] mod tests { - use std::ops::Range; + use std::{ops::Range, sync::Arc}; use crate::{ data_source::{kinds::data_manipulation::map::MapParseTo, types::BlockscoutMigrations}, @@ -197,6 +197,7 @@ mod tests { lines::{PredefinedMockSource, PseudoRandomMockRetrieve}, tests::point_construction::{d, d_v, d_v_double, d_v_int, dt, w_v_double, week_of}, types::timespans::{DateValue, Week, WeekValue}, + utils::MarkedDbConnection, MissingDatePolicy, }; @@ -319,7 +320,9 @@ mod tests { // 8-14, 15-21, 22-28 // db is not used in mock - let db = sea_orm::Database::connect("sqlite::memory:").await.unwrap(); + let db = MarkedDbConnection::in_memory(Arc::new( + sea_orm::Database::connect("sqlite::memory:").await.unwrap(), + )); let output: Vec> = TestedAverageSource::query_data( &UpdateContext { db: &db, @@ -367,7 +370,9 @@ mod tests { AverageLowerResolution; // db is not used in mock - let empty_db = sea_orm::Database::connect("sqlite::memory:").await.unwrap(); + let empty_db = MarkedDbConnection::in_memory(Arc::new( + sea_orm::Database::connect("sqlite::memory:").await.unwrap(), + )); let context = UpdateContext { db: &empty_db, @@ -417,7 +422,9 @@ mod tests { AverageLowerResolution; // db is not used in mock - let empty_db = sea_orm::Database::connect("sqlite::memory:").await.unwrap(); + let empty_db = MarkedDbConnection::in_memory(Arc::new( + sea_orm::Database::connect("sqlite::memory:").await.unwrap(), + )); let context = UpdateContext { db: &empty_db, @@ -463,7 +470,9 @@ mod tests { AverageLowerResolution; // db is not used in mock - let empty_db = sea_orm::Database::connect("sqlite::memory:").await.unwrap(); + let empty_db = MarkedDbConnection::in_memory(Arc::new( + sea_orm::Database::connect("sqlite::memory:").await.unwrap(), + )); let context = UpdateContext { db: &empty_db, diff --git a/stats/stats/src/data_source/kinds/data_manipulation/resolutions/last_value.rs b/stats/stats/src/data_source/kinds/data_manipulation/resolutions/last_value.rs index 35daaa90d..1789384a8 100644 --- a/stats/stats/src/data_source/kinds/data_manipulation/resolutions/last_value.rs +++ b/stats/stats/src/data_source/kinds/data_manipulation/resolutions/last_value.rs @@ -77,6 +77,8 @@ where #[cfg(test)] mod tests { + use std::sync::Arc; + use blockscout_metrics_tools::AggregateTimer; use pretty_assertions::assert_eq; @@ -87,6 +89,7 @@ mod tests { range::UniversalRange, tests::point_construction::{d_v_int, dt, w_v_int}, types::timespans::{DateValue, Week}, + utils::MarkedDbConnection, MissingDatePolicy, }; @@ -109,7 +112,9 @@ mod tests { type MockSourceWeekly = LastValueLowerResolution; // db is not used in mock - let empty_db = sea_orm::Database::connect("sqlite::memory:").await.unwrap(); + let empty_db = MarkedDbConnection::in_memory(Arc::new( + sea_orm::Database::connect("sqlite::memory:").await.unwrap(), + )); let context = UpdateContext { db: &empty_db, diff --git a/stats/stats/src/data_source/kinds/data_manipulation/resolutions/sum.rs b/stats/stats/src/data_source/kinds/data_manipulation/resolutions/sum.rs index 67896788c..24bd2ed27 100644 --- a/stats/stats/src/data_source/kinds/data_manipulation/resolutions/sum.rs +++ b/stats/stats/src/data_source/kinds/data_manipulation/resolutions/sum.rs @@ -97,6 +97,8 @@ where #[cfg(test)] mod tests { + use std::sync::Arc; + use blockscout_metrics_tools::AggregateTimer; use pretty_assertions::assert_eq; @@ -107,6 +109,7 @@ mod tests { range::UniversalRange, tests::point_construction::{d_v_int, dt, w_v_int}, types::timespans::{DateValue, Week}, + utils::MarkedDbConnection, MissingDatePolicy, }; @@ -129,7 +132,9 @@ mod tests { type MockSourceWeekly = SumLowerResolution; // db is not used in mock - let empty_db = sea_orm::Database::connect("sqlite::memory:").await.unwrap(); + let empty_db = MarkedDbConnection::in_memory(Arc::new( + sea_orm::Database::connect("sqlite::memory:").await.unwrap(), + )); let context = UpdateContext { db: &empty_db, diff --git a/stats/stats/src/data_source/kinds/local_db/mod.rs b/stats/stats/src/data_source/kinds/local_db/mod.rs index f45a82186..027a4174c 100644 --- a/stats/stats/src/data_source/kinds/local_db/mod.rs +++ b/stats/stats/src/data_source/kinds/local_db/mod.rs @@ -142,7 +142,7 @@ where cx: &UpdateContext<'_>, dependency_data_fetch_timer: &mut AggregateTimer, ) -> Result<(), UpdateError> { - let metadata = get_chart_metadata(cx.db, &ChartProps::key()).await?; + let metadata = get_chart_metadata(cx.db.connection.as_ref(), &ChartProps::key()).await?; if let Some(last_updated_at) = metadata.last_updated_at { if postgres_timestamps_eq(cx.time, last_updated_at) { // no need to perform update. @@ -163,13 +163,13 @@ where } } let chart_id = metadata.id; - let min_blockscout_block = get_min_block_blockscout(cx.blockscout) + let min_blockscout_block = get_min_block_blockscout(cx.blockscout.connection.as_ref()) .await .map_err(UpdateError::BlockscoutDB)?; let last_accurate_point = last_accurate_point::( chart_id, min_blockscout_block, - cx.db, + cx.db.connection.as_ref(), cx.force_full, ChartProps::approximate_trailing_points(), ChartProps::missing_date_policy(), @@ -185,7 +185,7 @@ where ) .await?; tracing::info!(chart =% ChartProps::key(), "updating chart metadata"); - Update::update_metadata(cx.db, chart_id, cx.time).await?; + Update::update_metadata(cx.db.connection.as_ref(), chart_id, cx.time).await?; Ok(()) } @@ -326,7 +326,7 @@ mod tests { DataSource, UpdateContext, UpdateParameters, }, gettable_const, - tests::{init_db::init_db_all, mock_blockscout::fill_mock_blockscout_data}, + tests::{init_db::init_marked_db_all, mock_blockscout::fill_mock_blockscout_data}, types::{timespans::DateValue, TimespanValue}, update_group::{SyncUpdateGroup, UpdateGroup}, ChartProperties, Named, UpdateError, @@ -376,7 +376,7 @@ mod tests { value: "0".to_owned(), }; let value = data.active_model(chart_id, Some(min_blockscout_block)); - insert_data_many(cx.db, vec![value]) + insert_data_many(cx.db.connection.as_ref(), vec![value]) .await .map_err(UpdateError::StatsDB)?; Ok(()) @@ -435,10 +435,11 @@ mod tests { #[ignore = "needs database to run"] async fn update_itself_is_triggered_once_per_group() { let _ = tracing_subscriber::fmt::try_init(); - let (db, blockscout) = init_db_all("update_itself_is_triggered_once_per_group").await; + let (db, blockscout) = + init_marked_db_all("update_itself_is_triggered_once_per_group").await; let current_time = DateTime::::from_str("2023-03-01T12:00:00Z").unwrap(); let current_date = current_time.date_naive(); - fill_mock_blockscout_data(&blockscout, current_date).await; + fill_mock_blockscout_data(blockscout.connection.as_ref(), current_date).await; let enabled = HashSet::from( [TestedChartProps::key(), ChartDependedOnTestedProps::key()].map(|l| l.to_owned()), ); @@ -449,7 +450,7 @@ mod tests { .collect(); let group = SyncUpdateGroup::new(&mutexes, Arc::new(TestUpdateGroup)).unwrap(); group - .create_charts_with_mutexes(&db, Some(current_time), &enabled) + .create_charts_with_mutexes(&db.connection.as_ref(), Some(current_time), &enabled) .await .unwrap(); diff --git a/stats/stats/src/data_source/kinds/local_db/parameters/query.rs b/stats/stats/src/data_source/kinds/local_db/parameters/query.rs index 68a38ae9d..476be1856 100644 --- a/stats/stats/src/data_source/kinds/local_db/parameters/query.rs +++ b/stats/stats/src/data_source/kinds/local_db/parameters/query.rs @@ -1,7 +1,6 @@ use std::{fmt::Debug, marker::PhantomData}; use chrono::{DateTime, Utc}; -use sea_orm::DatabaseConnection; use crate::{ charts::db_interaction::read::get_counter_data, @@ -9,6 +8,7 @@ use crate::{ get_line_chart_data, range::UniversalRange, types::{timespans::DateValue, ExtendedTimespanValue, Timespan}, + utils::MarkedDbConnection, ChartProperties, RequestedPointsLimit, UpdateError, }; @@ -49,7 +49,7 @@ where let start = start.map(|s| C::Resolution::from_date(s.date_naive())); let end = end.map(|e| C::Resolution::from_date(e.date_naive())); let values = get_line_chart_data::( - cx.db, + cx.db.connection.as_ref(), &C::name(), start, end, @@ -76,7 +76,7 @@ impl QueryBehaviour for DefaultQueryLast { _fill_missing_dates: bool, ) -> Result { let value = get_counter_data( - cx.db, + cx.db.connection.as_ref(), &C::name(), Some(cx.time.date_naive()), C::missing_date_policy(), @@ -92,7 +92,7 @@ impl QueryBehaviour for DefaultQueryLast { #[trait_variant::make(Send)] pub trait ValueEstimation { - async fn estimate(blockscout: &DatabaseConnection) -> Result, UpdateError>; + async fn estimate(blockscout: &MarkedDbConnection) -> Result, UpdateError>; } pub struct QueryLastWithEstimationFallback(PhantomData<(E, C)>) @@ -113,14 +113,17 @@ where _points_limit: Option, _fill_missing_dates: bool, ) -> Result { - let value = get_counter_data( - cx.db, + let value = match get_counter_data( + cx.db.connection.as_ref(), &C::name(), Some(cx.time.date_naive()), C::missing_date_policy(), ) .await? - .unwrap_or(E::estimate(cx.blockscout).await?); + { + Some(v) => v, + None => E::estimate(cx.blockscout).await?, + }; Ok(value) } } @@ -132,13 +135,12 @@ mod tests { use chrono::NaiveDate; use entity::sea_orm_active_enums::ChartType; use pretty_assertions::assert_eq; - use sea_orm::DatabaseConnection; use super::*; use crate::{ data_source::{types::BlockscoutMigrations, UpdateContext, UpdateParameters}, - tests::init_db::init_db_all, + tests::init_db::init_marked_db_all, types::timespans::DateValue, MissingDatePolicy, Named, UpdateError, }; @@ -147,7 +149,7 @@ mod tests { #[ignore = "needs database to run"] async fn fallback_query_works() { let _ = tracing_subscriber::fmt::try_init(); - let (db, blockscout) = init_db_all("fallback_query_works").await; + let (db, blockscout) = init_marked_db_all("fallback_query_works").await; let current_time = chrono::DateTime::from_str("2023-03-01T12:00:00Z").unwrap(); let parameters = UpdateParameters { @@ -170,7 +172,7 @@ mod tests { impl ValueEstimation for TestFallback { async fn estimate( - _blockscout: &DatabaseConnection, + _blockscout: &MarkedDbConnection, ) -> Result, UpdateError> { Ok(expected_estimate()) } diff --git a/stats/stats/src/data_source/kinds/local_db/parameters/update/batching/mod.rs b/stats/stats/src/data_source/kinds/local_db/parameters/update/batching/mod.rs index cbfa24996..13ac7c346 100644 --- a/stats/stats/src/data_source/kinds/local_db/parameters/update/batching/mod.rs +++ b/stats/stats/src/data_source/kinds/local_db/parameters/update/batching/mod.rs @@ -69,7 +69,7 @@ where let update_range_start = match update_from { Some(d) => d, None => ChartProps::Resolution::from_date( - get_min_date_blockscout(cx.blockscout) + get_min_date_blockscout(cx.blockscout.connection.as_ref()) .await .map(|time| time.date()) .map_err(UpdateError::BlockscoutDB)?, @@ -107,7 +107,12 @@ where ) .await?; // for query in `get_previous_step_last_point` to work correctly - Self::update_metadata(cx.db, chart_id, range.into_date_time_range().end).await?; + Self::update_metadata( + cx.db.connection.as_ref(), + chart_id, + range.into_date_time_range().end, + ) + .await?; let elapsed: std::time::Duration = now.elapsed(); tracing::info!( found =? found, @@ -179,7 +184,7 @@ where let resolution_data = ResolutionDep::query_data(cx, query_range, dependency_data_fetch_timer).await?; let found = BatchStep::batch_update_values_step_with( - cx.db, + cx.db.connection.as_ref(), chart_id, cx.time, min_blockscout_block, diff --git a/stats/stats/src/data_source/kinds/local_db/parameters/update/point.rs b/stats/stats/src/data_source/kinds/local_db/parameters/update/point.rs index 20a3f8e78..774bc348c 100644 --- a/stats/stats/src/data_source/kinds/local_db/parameters/update/point.rs +++ b/stats/stats/src/data_source/kinds/local_db/parameters/update/point.rs @@ -28,7 +28,7 @@ where // range doesn't make sense there; thus is not used let data = MainDep::query_data(cx, UniversalRange::full(), remote_fetch_timer).await?; let value = data.active_model(chart_id, Some(min_blockscout_block)); - insert_data_many(cx.db, vec![value]) + insert_data_many(cx.db.connection.as_ref(), vec![value]) .await .map_err(UpdateError::StatsDB)?; Ok(()) diff --git a/stats/stats/src/data_source/kinds/remote_db/query/all.rs b/stats/stats/src/data_source/kinds/remote_db/query/all.rs index de30f0ef5..acdf2436d 100644 --- a/stats/stats/src/data_source/kinds/remote_db/query/all.rs +++ b/stats/stats/src/data_source/kinds/remote_db/query/all.rs @@ -60,7 +60,7 @@ where data_source_query_range_to_db_statement_range::(cx, range).await?; let query = S::get_statement(query_range, &cx.blockscout_applied_migrations); let mut data = TimespanValue::::find_by_statement(query) - .all(cx.blockscout) + .all(cx.blockscout.connection.as_ref()) .await .map_err(UpdateError::BlockscoutDB)?; // linear time for sorted sequences diff --git a/stats/stats/src/data_source/kinds/remote_db/query/each.rs b/stats/stats/src/data_source/kinds/remote_db/query/each.rs index e1c2c4b91..13333b9eb 100644 --- a/stats/stats/src/data_source/kinds/remote_db/query/each.rs +++ b/stats/stats/src/data_source/kinds/remote_db/query/each.rs @@ -63,7 +63,7 @@ where for point_range in points { let query = S::get_statement(point_range.clone(), &cx.blockscout_applied_migrations); let point_value = ValueWrapper::::find_by_statement(query) - .one(cx.blockscout) + .one(cx.blockscout.connection.as_ref()) .await .map_err(UpdateError::BlockscoutDB)?; if let Some(ValueWrapper { value }) = point_value { diff --git a/stats/stats/src/data_source/kinds/remote_db/query/one.rs b/stats/stats/src/data_source/kinds/remote_db/query/one.rs index 96f781e64..aca5d1a68 100644 --- a/stats/stats/src/data_source/kinds/remote_db/query/one.rs +++ b/stats/stats/src/data_source/kinds/remote_db/query/one.rs @@ -45,7 +45,7 @@ where ) -> Result, UpdateError> { let query = S::get_statement(&cx.blockscout_applied_migrations); let data = TimespanValue::::find_by_statement(query) - .one(cx.blockscout) + .one(cx.blockscout.connection.as_ref()) .await .map_err(UpdateError::BlockscoutDB)? .ok_or_else(|| UpdateError::Internal("query returned nothing".into()))?; diff --git a/stats/stats/src/data_source/tests.rs b/stats/stats/src/data_source/tests.rs index f1eb97fd8..480d7cbf7 100644 --- a/stats/stats/src/data_source/tests.rs +++ b/stats/stats/src/data_source/tests.rs @@ -34,7 +34,7 @@ use crate::{ types::BlockscoutMigrations, }, define_and_impl_resolution_properties, - tests::{init_db::init_db_all, mock_blockscout::fill_mock_blockscout_data}, + tests::{init_db::init_marked_db_all, mock_blockscout::fill_mock_blockscout_data}, types::timespans::{DateValue, Month, Week, Year}, update_group::{SyncUpdateGroup, UpdateGroup}, utils::{produce_filter_and_values, sql_with_range_filter_opt}, @@ -267,10 +267,10 @@ construct_update_group!(ExampleUpdateGroup { #[ignore = "needs database to run"] async fn update_examples() { let _ = tracing_subscriber::fmt::try_init(); - let (db, blockscout) = init_db_all("update_examples").await; + let (db, blockscout) = init_marked_db_all("update_examples").await; let current_time = DateTime::::from_str("2023-03-01T12:00:00Z").unwrap(); let current_date = current_time.date_naive(); - fill_mock_blockscout_data(&blockscout, current_date).await; + fill_mock_blockscout_data(blockscout.connection.as_ref(), current_date).await; let enabled = HashSet::from( [ NewContractsChartProperties::key(), @@ -290,7 +290,7 @@ async fn update_examples() { .collect(); let group = SyncUpdateGroup::new(&mutexes, Arc::new(ExampleUpdateGroup)).unwrap(); group - .create_charts_with_mutexes(&db, None, &enabled) + .create_charts_with_mutexes(db.connection.as_ref(), None, &enabled) .await .unwrap(); diff --git a/stats/stats/src/data_source/types.rs b/stats/stats/src/data_source/types.rs index 56e3b3486..58dc2bdd3 100644 --- a/stats/stats/src/data_source/types.rs +++ b/stats/stats/src/data_source/types.rs @@ -3,10 +3,12 @@ use chrono::Utc; use sea_orm::{DatabaseConnection, DbErr, EntityTrait, FromQueryResult, QueryOrder, Statement}; use tracing::warn; +use crate::utils::MarkedDbConnection; + #[derive(Clone)] pub struct UpdateParameters<'a> { - pub db: &'a DatabaseConnection, - pub blockscout: &'a DatabaseConnection, + pub db: &'a MarkedDbConnection, + pub blockscout: &'a MarkedDbConnection, pub blockscout_applied_migrations: BlockscoutMigrations, /// If `None`, it will be measured at the start of update /// (i.e. after taking mutexes) @@ -17,8 +19,8 @@ pub struct UpdateParameters<'a> { #[derive(Clone)] pub struct UpdateContext<'a> { - pub db: &'a DatabaseConnection, - pub blockscout: &'a DatabaseConnection, + pub db: &'a MarkedDbConnection, + pub blockscout: &'a MarkedDbConnection, pub blockscout_applied_migrations: BlockscoutMigrations, /// Update time pub time: chrono::DateTime, diff --git a/stats/stats/src/tests/init_db.rs b/stats/stats/src/tests/init_db.rs index 655511e45..29473a678 100644 --- a/stats/stats/src/tests/init_db.rs +++ b/stats/stats/src/tests/init_db.rs @@ -1,5 +1,7 @@ use blockscout_service_launcher::test_database::TestDbGuard; +use crate::utils::MarkedDbConnection; + pub async fn init_db_all(name: &str) -> (TestDbGuard, TestDbGuard) { let db = init_db(name).await; let blockscout = @@ -11,3 +13,11 @@ pub async fn init_db_all(name: &str) -> (TestDbGuard, TestDbGuard) { pub async fn init_db(name: &str) -> TestDbGuard { TestDbGuard::new::(name).await } + +pub async fn init_marked_db_all(name: &str) -> (MarkedDbConnection, MarkedDbConnection) { + let (db, blockscout) = init_db_all(name).await; + ( + MarkedDbConnection::from_test_db(&db).unwrap(), + MarkedDbConnection::from_test_db(&blockscout).unwrap(), + ) +} diff --git a/stats/stats/src/tests/simple_test.rs b/stats/stats/src/tests/simple_test.rs index c9e872fb1..2bac23980 100644 --- a/stats/stats/src/tests/simple_test.rs +++ b/stats/stats/src/tests/simple_test.rs @@ -1,4 +1,7 @@ -use super::{init_db::init_db_all, mock_blockscout::fill_mock_blockscout_data}; +use super::{ + init_db::{init_db_all, init_marked_db_all}, + mock_blockscout::fill_mock_blockscout_data, +}; use crate::{ data_source::{ source::DataSource, @@ -8,6 +11,7 @@ use crate::{ query_dispatch::QuerySerialized, range::UniversalRange, types::{timespans::DateValue, Timespan}, + utils::MarkedDbConnection, ChartProperties, MissingDatePolicy, }; use blockscout_service_launcher::test_database::TestDbGuard; @@ -77,8 +81,8 @@ where fill_mock_blockscout_data(&blockscout, current_date).await; let mut parameters = UpdateParameters { - db: &db, - blockscout: &blockscout, + db: &MarkedDbConnection::from_test_db(&db).unwrap(), + blockscout: &MarkedDbConnection::from_test_db(&blockscout).unwrap(), blockscout_applied_migrations: migrations, update_time_override: Some(current_time), force_full: true, @@ -120,8 +124,8 @@ where /// /// Tests that force update with existing data works correctly pub async fn dirty_force_update_and_check( - db: &DatabaseConnection, - blockscout: &DatabaseConnection, + db: &TestDbGuard, + blockscout: &TestDbGuard, expected: Vec<(&str, &str)>, update_time_override: Option>, ) where @@ -136,8 +140,8 @@ pub async fn dirty_force_update_and_check( let approximate_trailing_points = C::approximate_trailing_points(); let parameters = UpdateParameters { - db, - blockscout, + db: &MarkedDbConnection::from_test_db(&db).unwrap(), + blockscout: &MarkedDbConnection::from_test_db(&blockscout).unwrap(), blockscout_applied_migrations: BlockscoutMigrations::latest(), update_time_override: Some(current_time), force_full: true, @@ -230,12 +234,14 @@ async fn ranged_test_chart_inner( { let _ = tracing_subscriber::fmt::try_init(); let expected = map_str_tuple_to_owned(expected); - let (db, blockscout) = init_db_all(test_name).await; + let (db, blockscout) = init_marked_db_all(test_name).await; let max_time = DateTime::::from_str("2023-03-01T12:00:00Z").unwrap(); let current_time = update_time.map(|t| t.and_utc()).unwrap_or(max_time); let max_date = max_time.date_naive(); - C::init_recursively(&db, ¤t_time).await.unwrap(); - fill_mock_blockscout_data(&blockscout, max_date).await; + C::init_recursively(db.connection.as_ref(), ¤t_time) + .await + .unwrap(); + fill_mock_blockscout_data(blockscout.connection.as_ref(), max_date).await; let policy = C::missing_date_policy(); let approximate_trailing_points = C::approximate_trailing_points(); @@ -250,7 +256,7 @@ async fn ranged_test_chart_inner( C::update_recursively(&cx).await.unwrap(); assert_eq!( &get_chart::( - &db, + db.connection.as_ref(), Some(from.clone()), Some(to.clone()), policy, @@ -266,7 +272,7 @@ async fn ranged_test_chart_inner( C::update_recursively(&cx).await.unwrap(); assert_eq!( &get_chart::( - &db, + db.connection.as_ref(), Some(from), Some(to), policy, @@ -358,8 +364,8 @@ async fn simple_test_counter_inner( fill_mock_blockscout_data(&blockscout, max_date).await; let mut parameters = UpdateParameters { - db: &db, - blockscout: &blockscout, + db: &MarkedDbConnection::from_test_db(&db).unwrap(), + blockscout: &MarkedDbConnection::from_test_db(&blockscout).unwrap(), blockscout_applied_migrations: migrations, update_time_override: Some(current_time), force_full: true, diff --git a/stats/stats/src/utils.rs b/stats/stats/src/utils.rs index 75deaf3bb..a3917bae2 100644 --- a/stats/stats/src/utils.rs +++ b/stats/stats/src/utils.rs @@ -1,7 +1,8 @@ //! Common utilities used across statistics -use std::ops::Range; +use std::{ops::Range, sync::Arc}; +use blockscout_service_launcher::test_database::TestDbGuard; use chrono::{DateTime, NaiveDate, NaiveTime, Utc}; use sea_orm::Value; @@ -13,6 +14,35 @@ pub fn day_start(date: &NaiveDate) -> DateTime { .and_utc() } +#[derive(Debug, Clone)] +pub struct MarkedDbConnection { + pub connection: Arc, + pub db_name: String, +} + +impl MarkedDbConnection { + pub fn from_test_db(guard: &TestDbGuard) -> Option { + Some(Self { + connection: guard.client(), + db_name: guard.db_url().split("/").last()?.to_owned(), + }) + } + + pub fn main_connection(inner: Arc) -> Self { + Self { + connection: inner, + db_name: "main".to_owned(), + } + } + + pub fn in_memory(inner: Arc) -> Self { + Self { + connection: inner, + db_name: "in_memory".to_owned(), + } + } +} + /// Used inside [`sql_with_range_filter_opt`] /// /// `filter_arg_number_start = len(arg)+1 // (length of other args + 1)` From 0ca5c7bbdb7795ba9a0ac13d7128d8877a799171 Mon Sep 17 00:00:00 2001 From: Kirill Ivanov Date: Wed, 4 Dec 2024 17:04:51 +0900 Subject: [PATCH 19/41] fix tests --- stats/stats/src/charts/db_interaction/read.rs | 8 +++++--- stats/stats/src/update_group.rs | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/stats/stats/src/charts/db_interaction/read.rs b/stats/stats/src/charts/db_interaction/read.rs index af00a45bb..c84d733d5 100644 --- a/stats/stats/src/charts/db_interaction/read.rs +++ b/stats/stats/src/charts/db_interaction/read.rs @@ -679,7 +679,7 @@ mod tests { tests::{ init_db::{init_db, init_db_all}, mock_blockscout::fill_mock_blockscout_data, - point_construction::{d, month_of}, + point_construction::{d, dt, month_of}, simple_test::get_counter, }, types::timespans::Month, @@ -831,16 +831,18 @@ mod tests { let db = MarkedDbConnection::from_test_db(&init_db("get_counters_mock").await).unwrap(); insert_mock_data(&db.connection).await; + let current_time = dt("2022-11-12T08:08:08").and_utc(); + let date = current_time.date_naive(); let cx = UpdateContext::from_params_now_or_override(UpdateParameters { db: &db, // shouldn't use this because mock data contains total blocks value blockscout: &db, blockscout_applied_migrations: BlockscoutMigrations::latest(), - update_time_override: None, + update_time_override: Some(current_time), force_full: false, }); assert_eq!( - value("2022-11-12", "1350"), + value(&date.to_string(), "1350"), get_counter::(&cx).await ); } diff --git a/stats/stats/src/update_group.rs b/stats/stats/src/update_group.rs index 59c187ba3..ec8793c19 100644 --- a/stats/stats/src/update_group.rs +++ b/stats/stats/src/update_group.rs @@ -207,7 +207,7 @@ pub trait UpdateGroup: core::fmt::Debug { /// # }, /// # types::{UpdateContext, UpdateParameters, BlockscoutMigrations}, /// # }; -/// # use chrono::NaiveDate; +/// # use chrono::{NaiveDate, DateTime, Utc}; /// # use entity::sea_orm_active_enums::ChartType; /// # use std::ops::Range; /// # use sea_orm::Statement; From 8e88661e541e5c7f9632307ec69805fefd956276 Mon Sep 17 00:00:00 2001 From: Kirill Ivanov Date: Wed, 4 Dec 2024 17:05:55 +0900 Subject: [PATCH 20/41] clippy --- stats/stats/src/data_source/kinds/local_db/mod.rs | 2 +- stats/stats/src/tests/simple_test.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/stats/stats/src/data_source/kinds/local_db/mod.rs b/stats/stats/src/data_source/kinds/local_db/mod.rs index 027a4174c..e73437f7f 100644 --- a/stats/stats/src/data_source/kinds/local_db/mod.rs +++ b/stats/stats/src/data_source/kinds/local_db/mod.rs @@ -450,7 +450,7 @@ mod tests { .collect(); let group = SyncUpdateGroup::new(&mutexes, Arc::new(TestUpdateGroup)).unwrap(); group - .create_charts_with_mutexes(&db.connection.as_ref(), Some(current_time), &enabled) + .create_charts_with_mutexes(db.connection.as_ref(), Some(current_time), &enabled) .await .unwrap(); diff --git a/stats/stats/src/tests/simple_test.rs b/stats/stats/src/tests/simple_test.rs index 2bac23980..482b80855 100644 --- a/stats/stats/src/tests/simple_test.rs +++ b/stats/stats/src/tests/simple_test.rs @@ -140,8 +140,8 @@ pub async fn dirty_force_update_and_check( let approximate_trailing_points = C::approximate_trailing_points(); let parameters = UpdateParameters { - db: &MarkedDbConnection::from_test_db(&db).unwrap(), - blockscout: &MarkedDbConnection::from_test_db(&blockscout).unwrap(), + db: &MarkedDbConnection::from_test_db(db).unwrap(), + blockscout: &MarkedDbConnection::from_test_db(blockscout).unwrap(), blockscout_applied_migrations: BlockscoutMigrations::latest(), update_time_override: Some(current_time), force_full: true, From 77344a2845d406e4a9831cf22f18be90d6c38c03 Mon Sep 17 00:00:00 2001 From: Kirill Ivanov Date: Wed, 4 Dec 2024 19:00:58 +0900 Subject: [PATCH 21/41] manual usage of cache instead of proc macro to support configurable timeout --- .../stats/src/charts/counters/total_blocks.rs | 40 ++++++++++++++----- stats/stats/src/charts/db_interaction/read.rs | 10 ++--- 2 files changed, 35 insertions(+), 15 deletions(-) diff --git a/stats/stats/src/charts/counters/total_blocks.rs b/stats/stats/src/charts/counters/total_blocks.rs index 2adce75e0..5e246b7da 100644 --- a/stats/stats/src/charts/counters/total_blocks.rs +++ b/stats/stats/src/charts/counters/total_blocks.rs @@ -1,5 +1,7 @@ +use std::sync::OnceLock; + use crate::{ - charts::db_interaction::read::get_estimated_table_rows, + charts::db_interaction::read::query_estimated_table_rows, data_source::{ kinds::{ local_db::{parameters::ValueEstimation, DirectPointLocalDbChartSourceWithEstimate}, @@ -14,9 +16,11 @@ use crate::{ }; use blockscout_db::entity::blocks; +use cached::Cached; use chrono::{DateTime, NaiveDate, NaiveDateTime, Utc}; use entity::sea_orm_active_enums::ChartType; use sea_orm::{prelude::*, sea_query::Expr, FromQueryResult, QuerySelect}; +use tokio::sync::Mutex; #[derive(FromQueryResult)] struct TotalBlocksData { @@ -75,20 +79,36 @@ impl ChartProperties for Properties { pub struct CachedBlocksEstimation; +const TOTAL_BLOCKS_ESTIMATION_CACHE_LIVENESS_SEC_DEFAULT: u64 = 10; +pub static TOTAL_BLOCKS_ESTIMATION_CACHE_LIVENESS_SEC: OnceLock = OnceLock::new(); +static CACHED_BLOCKS_ESTIMATION: OnceLock>> = OnceLock::new(); + impl ValueEstimation for CachedBlocksEstimation { async fn estimate(blockscout: &MarkedDbConnection) -> Result, UpdateError> { - #[cached::proc_macro::cached( - time = 60, - key = "String", - convert = r#"{ String::from(_db_id) }"#, - sync_writes = true, - result = true - )] async fn cached_blocks_estimation( blockscout: &DatabaseConnection, - _db_id: &str, + db_id: &str, ) -> Result, DbErr> { - get_estimated_table_rows(blockscout, blocks::Entity.table_name()).await + let mut cache = CACHED_BLOCKS_ESTIMATION + .get_or_init(|| { + Mutex::new(cached::TimedCache::with_lifespan_and_refresh( + *TOTAL_BLOCKS_ESTIMATION_CACHE_LIVENESS_SEC + .get_or_init(|| TOTAL_BLOCKS_ESTIMATION_CACHE_LIVENESS_SEC_DEFAULT), + false, + )) + }) + .lock() + .await; + if let Some(cached_estimate) = cache.cache_get(db_id) { + return Ok(Some(*cached_estimate)); + } + + let query_result = + query_estimated_table_rows(blockscout, blocks::Entity.table_name()).await; + if let Ok(Some(estimate)) = &query_result { + cache.cache_set(db_id.to_string(), *estimate); + } + query_result } let now = Utc::now(); diff --git a/stats/stats/src/charts/db_interaction/read.rs b/stats/stats/src/charts/db_interaction/read.rs index c84d733d5..271ed4724 100644 --- a/stats/stats/src/charts/db_interaction/read.rs +++ b/stats/stats/src/charts/db_interaction/read.rs @@ -435,7 +435,7 @@ struct CountEstimate { /// `None` means either that /// - db hasn't been initialized before /// - `table_name` wasn't found -pub async fn get_estimated_table_rows( +pub async fn query_estimated_table_rows( blockscout: &DatabaseConnection, table_name: &str, ) -> Result, DbErr> { @@ -1316,7 +1316,7 @@ mod tests { .await .unwrap(); - let blocks_estimate = get_estimated_table_rows(&blockscout, blocks::Entity.table_name()) + let blocks_estimate = query_estimated_table_rows(&blockscout, blocks::Entity.table_name()) .await .unwrap() .unwrap(); @@ -1326,7 +1326,7 @@ mod tests { assert!(blocks_estimate < 30); assert!( - get_estimated_table_rows( + query_estimated_table_rows( &blockscout, blockscout_db::entity::addresses::Entity.table_name() ) @@ -1337,7 +1337,7 @@ mod tests { ); assert!( - get_estimated_table_rows( + query_estimated_table_rows( &blockscout, blockscout_db::entity::transactions::Entity.table_name() ) @@ -1348,7 +1348,7 @@ mod tests { ); assert!( - get_estimated_table_rows( + query_estimated_table_rows( &blockscout, blockscout_db::entity::smart_contracts::Entity.table_name() ) From b3b7e156477f3ddbccec715c76f0f8f358562339 Mon Sep 17 00:00:00 2001 From: Kirill Ivanov Date: Wed, 4 Dec 2024 19:34:11 +0900 Subject: [PATCH 22/41] fix non-test build --- stats/stats/src/utils.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/stats/stats/src/utils.rs b/stats/stats/src/utils.rs index a3917bae2..3e0b9e03b 100644 --- a/stats/stats/src/utils.rs +++ b/stats/stats/src/utils.rs @@ -1,10 +1,10 @@ //! Common utilities used across statistics -use std::{ops::Range, sync::Arc}; - +#[cfg(test)] use blockscout_service_launcher::test_database::TestDbGuard; use chrono::{DateTime, NaiveDate, NaiveTime, Utc}; use sea_orm::Value; +use std::{ops::Range, sync::Arc}; // this const is not public in `chrono` for some reason pub const NANOS_PER_SEC: i32 = 1_000_000_000; @@ -21,6 +21,7 @@ pub struct MarkedDbConnection { } impl MarkedDbConnection { + #[cfg(test)] pub fn from_test_db(guard: &TestDbGuard) -> Option { Some(Self { connection: guard.client(), From 715d0ee15d54f13e4a8a8d92e7385454aa5bb29e Mon Sep 17 00:00:00 2001 From: Kirill Ivanov Date: Wed, 4 Dec 2024 19:39:40 +0900 Subject: [PATCH 23/41] explain once lock --- stats/stats/src/charts/counters/total_blocks.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/stats/stats/src/charts/counters/total_blocks.rs b/stats/stats/src/charts/counters/total_blocks.rs index 5e246b7da..5e006e1d2 100644 --- a/stats/stats/src/charts/counters/total_blocks.rs +++ b/stats/stats/src/charts/counters/total_blocks.rs @@ -80,6 +80,7 @@ impl ChartProperties for Properties { pub struct CachedBlocksEstimation; const TOTAL_BLOCKS_ESTIMATION_CACHE_LIVENESS_SEC_DEFAULT: u64 = 10; +// so that it can be added to settings if necessary pub static TOTAL_BLOCKS_ESTIMATION_CACHE_LIVENESS_SEC: OnceLock = OnceLock::new(); static CACHED_BLOCKS_ESTIMATION: OnceLock>> = OnceLock::new(); From e5bd7dcc50cd8ffcda347fa94901823251090bf4 Mon Sep 17 00:00:00 2001 From: Kirill Ivanov Date: Wed, 4 Dec 2024 19:48:06 +0900 Subject: [PATCH 24/41] fmt --- stats/stats/src/charts/lines/average_block_rewards.rs | 3 +-- stats/stats/src/charts/lines/average_block_size.rs | 3 +-- stats/stats/src/charts/lines/average_gas_limit.rs | 3 +-- stats/stats/src/charts/lines/average_gas_price.rs | 3 +-- stats/stats/src/charts/lines/average_txn_fee.rs | 3 +-- stats/stats/src/charts/lines/gas_used_growth.rs | 3 +-- stats/stats/src/charts/lines/native_coin_supply.rs | 3 +-- stats/stats/src/charts/lines/new_block_rewards.rs | 3 +-- stats/stats/src/charts/lines/new_blocks.rs | 3 +-- stats/stats/src/charts/lines/new_contracts.rs | 3 +-- stats/stats/src/charts/lines/new_native_coin_transfers.rs | 3 +-- stats/stats/src/charts/lines/new_txns.rs | 3 +-- stats/stats/src/charts/lines/new_verified_contracts.rs | 3 +-- stats/stats/src/charts/lines/txns_fee.rs | 3 +-- stats/stats/src/charts/lines/txns_success_rate.rs | 3 +-- stats/stats/src/data_source/kinds/auxiliary/cumulative.rs | 5 +---- .../stats/src/data_source/kinds/data_manipulation/delta.rs | 7 +------ .../src/data_source/kinds/data_manipulation/sum_point.rs | 5 +---- 18 files changed, 18 insertions(+), 44 deletions(-) diff --git a/stats/stats/src/charts/lines/average_block_rewards.rs b/stats/stats/src/charts/lines/average_block_rewards.rs index 9b6c849b1..d14357f73 100644 --- a/stats/stats/src/charts/lines/average_block_rewards.rs +++ b/stats/stats/src/charts/lines/average_block_rewards.rs @@ -24,8 +24,7 @@ use crate::{ ChartProperties, Named, }; -use chrono::NaiveDate; -use chrono::{DateTime, Utc}; +use chrono::{DateTime, NaiveDate, Utc}; use entity::sea_orm_active_enums::ChartType; use sea_orm::{DbBackend, Statement}; diff --git a/stats/stats/src/charts/lines/average_block_size.rs b/stats/stats/src/charts/lines/average_block_size.rs index 06d216ef1..965b81645 100644 --- a/stats/stats/src/charts/lines/average_block_size.rs +++ b/stats/stats/src/charts/lines/average_block_size.rs @@ -24,8 +24,7 @@ use crate::{ ChartProperties, Named, }; -use chrono::NaiveDate; -use chrono::{DateTime, Utc}; +use chrono::{DateTime, NaiveDate, Utc}; use entity::sea_orm_active_enums::ChartType; use sea_orm::{DbBackend, Statement}; diff --git a/stats/stats/src/charts/lines/average_gas_limit.rs b/stats/stats/src/charts/lines/average_gas_limit.rs index 291ccf940..81385b354 100644 --- a/stats/stats/src/charts/lines/average_gas_limit.rs +++ b/stats/stats/src/charts/lines/average_gas_limit.rs @@ -24,8 +24,7 @@ use crate::{ ChartProperties, Named, }; -use chrono::NaiveDate; -use chrono::{DateTime, Utc}; +use chrono::{DateTime, NaiveDate, Utc}; use entity::sea_orm_active_enums::ChartType; use sea_orm::{DbBackend, Statement}; diff --git a/stats/stats/src/charts/lines/average_gas_price.rs b/stats/stats/src/charts/lines/average_gas_price.rs index 80d380765..bf1c1c0c5 100644 --- a/stats/stats/src/charts/lines/average_gas_price.rs +++ b/stats/stats/src/charts/lines/average_gas_price.rs @@ -24,8 +24,7 @@ use crate::{ ChartProperties, Named, }; -use chrono::NaiveDate; -use chrono::{DateTime, Utc}; +use chrono::{DateTime, NaiveDate, Utc}; use entity::sea_orm_active_enums::ChartType; use sea_orm::{DbBackend, Statement}; diff --git a/stats/stats/src/charts/lines/average_txn_fee.rs b/stats/stats/src/charts/lines/average_txn_fee.rs index 431d8a2c4..7470243a7 100644 --- a/stats/stats/src/charts/lines/average_txn_fee.rs +++ b/stats/stats/src/charts/lines/average_txn_fee.rs @@ -26,8 +26,7 @@ use crate::{ ChartProperties, Named, }; -use chrono::NaiveDate; -use chrono::{DateTime, Utc}; +use chrono::{DateTime, NaiveDate, Utc}; use entity::sea_orm_active_enums::ChartType; use sea_orm::{DbBackend, Statement}; diff --git a/stats/stats/src/charts/lines/gas_used_growth.rs b/stats/stats/src/charts/lines/gas_used_growth.rs index 0421a4a44..6b9cc3bfe 100644 --- a/stats/stats/src/charts/lines/gas_used_growth.rs +++ b/stats/stats/src/charts/lines/gas_used_growth.rs @@ -24,8 +24,7 @@ use crate::{ ChartProperties, MissingDatePolicy, Named, UpdateError, }; -use chrono::NaiveDate; -use chrono::{DateTime, Utc}; +use chrono::{DateTime, NaiveDate, Utc}; use entity::sea_orm_active_enums::ChartType; use rust_decimal::Decimal; use sea_orm::{DbBackend, Statement}; diff --git a/stats/stats/src/charts/lines/native_coin_supply.rs b/stats/stats/src/charts/lines/native_coin_supply.rs index b3ff3188b..269259dce 100644 --- a/stats/stats/src/charts/lines/native_coin_supply.rs +++ b/stats/stats/src/charts/lines/native_coin_supply.rs @@ -23,8 +23,7 @@ use crate::{ ChartProperties, Named, }; -use chrono::NaiveDate; -use chrono::{DateTime, Utc}; +use chrono::{DateTime, NaiveDate, Utc}; use entity::sea_orm_active_enums::ChartType; use sea_orm::{DbBackend, Statement}; diff --git a/stats/stats/src/charts/lines/new_block_rewards.rs b/stats/stats/src/charts/lines/new_block_rewards.rs index b14f9f0a9..225610c99 100644 --- a/stats/stats/src/charts/lines/new_block_rewards.rs +++ b/stats/stats/src/charts/lines/new_block_rewards.rs @@ -26,8 +26,7 @@ use crate::{ ChartProperties, Named, }; -use chrono::NaiveDate; -use chrono::{DateTime, Utc}; +use chrono::{DateTime, NaiveDate, Utc}; use entity::sea_orm_active_enums::ChartType; use sea_orm::{DbBackend, Statement}; diff --git a/stats/stats/src/charts/lines/new_blocks.rs b/stats/stats/src/charts/lines/new_blocks.rs index 014c2d752..373745455 100644 --- a/stats/stats/src/charts/lines/new_blocks.rs +++ b/stats/stats/src/charts/lines/new_blocks.rs @@ -24,8 +24,7 @@ use crate::{ ChartProperties, Named, }; -use chrono::NaiveDate; -use chrono::{DateTime, Utc}; +use chrono::{DateTime, NaiveDate, Utc}; use entity::sea_orm_active_enums::ChartType; use sea_orm::{DbBackend, Statement}; diff --git a/stats/stats/src/charts/lines/new_contracts.rs b/stats/stats/src/charts/lines/new_contracts.rs index 0c7ec3d51..b6702b6be 100644 --- a/stats/stats/src/charts/lines/new_contracts.rs +++ b/stats/stats/src/charts/lines/new_contracts.rs @@ -24,8 +24,7 @@ use crate::{ ChartProperties, Named, }; -use chrono::NaiveDate; -use chrono::{DateTime, Utc}; +use chrono::{DateTime, NaiveDate, Utc}; use entity::sea_orm_active_enums::ChartType; use sea_orm::{DbBackend, Statement}; diff --git a/stats/stats/src/charts/lines/new_native_coin_transfers.rs b/stats/stats/src/charts/lines/new_native_coin_transfers.rs index db9874def..53fcc1cac 100644 --- a/stats/stats/src/charts/lines/new_native_coin_transfers.rs +++ b/stats/stats/src/charts/lines/new_native_coin_transfers.rs @@ -24,8 +24,7 @@ use crate::{ ChartProperties, Named, }; -use chrono::NaiveDate; -use chrono::{DateTime, Utc}; +use chrono::{DateTime, NaiveDate, Utc}; use entity::sea_orm_active_enums::ChartType; use sea_orm::{DbBackend, Statement}; diff --git a/stats/stats/src/charts/lines/new_txns.rs b/stats/stats/src/charts/lines/new_txns.rs index 6fec2a7f6..2f3b3fd26 100644 --- a/stats/stats/src/charts/lines/new_txns.rs +++ b/stats/stats/src/charts/lines/new_txns.rs @@ -24,8 +24,7 @@ use crate::{ ChartProperties, Named, }; -use chrono::NaiveDate; -use chrono::{DateTime, Utc}; +use chrono::{DateTime, NaiveDate, Utc}; use entity::sea_orm_active_enums::ChartType; use sea_orm::{DbBackend, Statement}; diff --git a/stats/stats/src/charts/lines/new_verified_contracts.rs b/stats/stats/src/charts/lines/new_verified_contracts.rs index a677918d4..c448c0f0c 100644 --- a/stats/stats/src/charts/lines/new_verified_contracts.rs +++ b/stats/stats/src/charts/lines/new_verified_contracts.rs @@ -24,8 +24,7 @@ use crate::{ ChartProperties, Named, }; -use chrono::NaiveDate; -use chrono::{DateTime, Utc}; +use chrono::{DateTime, NaiveDate, Utc}; use entity::sea_orm_active_enums::ChartType; use sea_orm::{DbBackend, Statement}; diff --git a/stats/stats/src/charts/lines/txns_fee.rs b/stats/stats/src/charts/lines/txns_fee.rs index 326ddd99d..18f363d02 100644 --- a/stats/stats/src/charts/lines/txns_fee.rs +++ b/stats/stats/src/charts/lines/txns_fee.rs @@ -26,8 +26,7 @@ use crate::{ ChartProperties, Named, }; -use chrono::NaiveDate; -use chrono::{DateTime, Utc}; +use chrono::{DateTime, NaiveDate, Utc}; use entity::sea_orm_active_enums::ChartType; use sea_orm::{DbBackend, Statement}; diff --git a/stats/stats/src/charts/lines/txns_success_rate.rs b/stats/stats/src/charts/lines/txns_success_rate.rs index bc122dd27..526b893d7 100644 --- a/stats/stats/src/charts/lines/txns_success_rate.rs +++ b/stats/stats/src/charts/lines/txns_success_rate.rs @@ -24,8 +24,7 @@ use crate::{ ChartProperties, Named, }; -use chrono::NaiveDate; -use chrono::{DateTime, Utc}; +use chrono::{DateTime, NaiveDate, Utc}; use entity::sea_orm_active_enums::ChartType; use sea_orm::{DbBackend, Statement}; diff --git a/stats/stats/src/data_source/kinds/auxiliary/cumulative.rs b/stats/stats/src/data_source/kinds/auxiliary/cumulative.rs index 3b13235eb..6fa509bc1 100644 --- a/stats/stats/src/data_source/kinds/auxiliary/cumulative.rs +++ b/stats/stats/src/data_source/kinds/auxiliary/cumulative.rs @@ -1,7 +1,4 @@ -use std::{ - marker::PhantomData, - ops::AddAssign, -}; +use std::{marker::PhantomData, ops::AddAssign}; use blockscout_metrics_tools::AggregateTimer; use chrono::{DateTime, Utc}; diff --git a/stats/stats/src/data_source/kinds/data_manipulation/delta.rs b/stats/stats/src/data_source/kinds/data_manipulation/delta.rs index b741289a8..d07762d70 100644 --- a/stats/stats/src/data_source/kinds/data_manipulation/delta.rs +++ b/stats/stats/src/data_source/kinds/data_manipulation/delta.rs @@ -3,12 +3,7 @@ //! //! I.e. chart "New accounts" is a delta of "Total accounts". -use std::{ - fmt::Display, - marker::PhantomData, - ops::SubAssign, - str::FromStr, -}; +use std::{fmt::Display, marker::PhantomData, ops::SubAssign, str::FromStr}; use blockscout_metrics_tools::AggregateTimer; use chrono::{DateTime, TimeDelta, Utc}; diff --git a/stats/stats/src/data_source/kinds/data_manipulation/sum_point.rs b/stats/stats/src/data_source/kinds/data_manipulation/sum_point.rs index e26345f5e..b8de19092 100644 --- a/stats/stats/src/data_source/kinds/data_manipulation/sum_point.rs +++ b/stats/stats/src/data_source/kinds/data_manipulation/sum_point.rs @@ -2,10 +2,7 @@ //! //! Sums all points from the other (vector) source. -use std::{ - marker::PhantomData, - ops::AddAssign, -}; +use std::{marker::PhantomData, ops::AddAssign}; use blockscout_metrics_tools::AggregateTimer; use chrono::{DateTime, Utc}; From 27a1f683056d0014faa844610e94a138b6cddaa5 Mon Sep 17 00:00:00 2001 From: Kirill Ivanov Date: Wed, 4 Dec 2024 19:48:11 +0900 Subject: [PATCH 25/41] fix test-utils build --- stats/stats/src/utils.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/stats/stats/src/utils.rs b/stats/stats/src/utils.rs index 3e0b9e03b..8220dffda 100644 --- a/stats/stats/src/utils.rs +++ b/stats/stats/src/utils.rs @@ -1,7 +1,5 @@ //! Common utilities used across statistics -#[cfg(test)] -use blockscout_service_launcher::test_database::TestDbGuard; use chrono::{DateTime, NaiveDate, NaiveTime, Utc}; use sea_orm::Value; use std::{ops::Range, sync::Arc}; @@ -21,8 +19,10 @@ pub struct MarkedDbConnection { } impl MarkedDbConnection { - #[cfg(test)] - pub fn from_test_db(guard: &TestDbGuard) -> Option { + #![cfg(any(feature = "test-utils", test))] + pub fn from_test_db( + guard: &blockscout_service_launcher::test_database::TestDbGuard, + ) -> Option { Some(Self { connection: guard.client(), db_name: guard.db_url().split("/").last()?.to_owned(), From 77586c38b3c643492d45655436f86a3d964212af Mon Sep 17 00:00:00 2001 From: Kirill Ivanov Date: Wed, 4 Dec 2024 19:57:06 +0900 Subject: [PATCH 26/41] rename update error to chart error for more accurate meaning --- stats/stats-server/src/read_service.rs | 14 ++++----- stats/stats/src/charts/chart.rs | 11 ++++--- .../src/charts/counters/average_block_time.rs | 10 +++---- stats/stats/src/charts/counters/mock.rs | 4 +-- .../stats/src/charts/counters/total_blocks.rs | 12 ++++---- .../src/charts/counters/total_contracts.rs | 6 ++-- .../src/charts/counters/yesterday_txns.rs | 8 ++--- stats/stats/src/charts/db_interaction/read.rs | 12 ++++---- .../stats/src/charts/lines/gas_used_growth.rs | 4 +-- stats/stats/src/charts/lines/mock.rs | 6 ++-- .../lines/native_coin_holders_growth.rs | 30 +++++++++---------- stats/stats/src/charts/lines/new_accounts.rs | 6 ++-- stats/stats/src/charts/mod.rs | 2 +- stats/stats/src/charts/query_dispatch.rs | 6 ++-- stats/stats/src/data_processing.rs | 8 ++--- .../data_source/kinds/auxiliary/cumulative.rs | 6 ++-- .../kinds/data_manipulation/delta.rs | 6 ++-- .../data_manipulation/filter_deducible.rs | 6 ++-- .../kinds/data_manipulation/last_point.rs | 6 ++-- .../kinds/data_manipulation/map/mod.rs | 8 ++--- .../kinds/data_manipulation/map/parse.rs | 12 ++++---- .../data_manipulation/map/strip_extension.rs | 6 ++-- .../kinds/data_manipulation/map/to_string.rs | 6 ++-- .../data_manipulation/resolutions/average.rs | 6 ++-- .../resolutions/last_value.rs | 6 ++-- .../data_manipulation/resolutions/sum.rs | 6 ++-- .../kinds/data_manipulation/sum_point.rs | 6 ++-- .../src/data_source/kinds/local_db/mod.rs | 16 +++++----- .../kinds/local_db/parameter_traits.rs | 10 +++---- .../kinds/local_db/parameters/query.rs | 16 +++++----- .../parameters/update/batching/mod.rs | 16 +++++----- .../update/batching/parameter_traits.rs | 4 +-- .../update/batching/parameters/cumulative.rs | 6 ++-- .../update/batching/parameters/mock.rs | 4 +-- .../update/batching/parameters/mod.rs | 6 ++-- .../kinds/local_db/parameters/update/point.rs | 6 ++-- .../src/data_source/kinds/remote_db/mod.rs | 8 ++--- .../data_source/kinds/remote_db/query/all.rs | 6 ++-- .../data_source/kinds/remote_db/query/each.rs | 6 ++-- .../data_source/kinds/remote_db/query/one.rs | 8 ++--- stats/stats/src/data_source/source.rs | 23 +++++++------- stats/stats/src/data_source/tests.rs | 4 +-- stats/stats/src/lib.rs | 2 +- stats/stats/src/range.rs | 4 +-- stats/stats/src/update_group.rs | 16 +++++----- stats/stats/src/utils.rs | 3 ++ 46 files changed, 192 insertions(+), 191 deletions(-) diff --git a/stats/stats-server/src/read_service.rs b/stats/stats-server/src/read_service.rs index c250b42d6..5784db94d 100644 --- a/stats/stats-server/src/read_service.rs +++ b/stats/stats-server/src/read_service.rs @@ -20,7 +20,7 @@ use stats::{ range::UniversalRange, types::Timespan, utils::{day_start, MarkedDbConnection}, - RequestedPointsLimit, ResolutionKind, UpdateError, + RequestedPointsLimit, ResolutionKind, ChartError, }; use stats_proto::blockscout::stats::v1 as proto_v1; use tonic::{Request, Response, Status}; @@ -63,10 +63,10 @@ impl From for ReadLimits { } } -fn map_update_error(err: UpdateError) -> Status { +fn map_update_error(err: ChartError) -> Status { match &err { - UpdateError::ChartNotFound(_) => Status::not_found(err.to_string()), - UpdateError::IntervalTooLarge { limit: _ } => Status::invalid_argument(err.to_string()), + ChartError::ChartNotFound(_) => Status::not_found(err.to_string()), + ChartError::IntervalTooLarge { limit: _ } => Status::invalid_argument(err.to_string()), _ => { tracing::error!(err = ?err, "internal read error"); Status::internal(err.to_string()) @@ -103,10 +103,10 @@ impl ReadService { range: UniversalRange>, points_limit: Option, query_time: DateTime, - ) -> Result { + ) -> Result { let migrations = BlockscoutMigrations::query_from_db(self.blockscout.connection.as_ref()) .await - .map_err(UpdateError::BlockscoutDB)?; + .map_err(ChartError::BlockscoutDB)?; let context = UpdateContext::from_params_now_or_override(UpdateParameters { db: &self.db, blockscout: &self.blockscout, @@ -146,7 +146,7 @@ impl ReadService { range: UniversalRange>, points_limit: Option, query_time: DateTime, - ) -> Result { + ) -> Result { let data = self .query_with_handle(query_handle, range, points_limit, query_time) .await?; diff --git a/stats/stats/src/charts/chart.rs b/stats/stats/src/charts/chart.rs index b3ea22ff1..8713a897f 100644 --- a/stats/stats/src/charts/chart.rs +++ b/stats/stats/src/charts/chart.rs @@ -17,9 +17,8 @@ use super::{ query_dispatch::{ChartTypeSpecifics, QuerySerialized, QuerySerializedDyn}, }; -// todo: rename to `ChartError` or similar #[derive(Error, Debug)] -pub enum UpdateError { +pub enum ChartError { #[error("blockscout database error: {0}")] BlockscoutDB(DbErr), #[error("stats database error: {0}")] @@ -32,12 +31,12 @@ pub enum UpdateError { Internal(String), } -impl From for UpdateError { +impl From for ChartError { fn from(read: ReadError) -> Self { match read { - ReadError::DB(db) => UpdateError::StatsDB(db), - ReadError::ChartNotFound(err) => UpdateError::ChartNotFound(err), - ReadError::IntervalTooLarge(limit) => UpdateError::IntervalTooLarge { limit }, + ReadError::DB(db) => ChartError::StatsDB(db), + ReadError::ChartNotFound(err) => ChartError::ChartNotFound(err), + ReadError::IntervalTooLarge(limit) => ChartError::IntervalTooLarge { limit }, } } } diff --git a/stats/stats/src/charts/counters/average_block_time.rs b/stats/stats/src/charts/counters/average_block_time.rs index bb9b05c81..67c76a1f9 100644 --- a/stats/stats/src/charts/counters/average_block_time.rs +++ b/stats/stats/src/charts/counters/average_block_time.rs @@ -12,7 +12,7 @@ use crate::{ range::UniversalRange, types::TimespanValue, utils::NANOS_PER_SEC, - ChartProperties, MissingDatePolicy, Named, UpdateError, + ChartProperties, MissingDatePolicy, Named, ChartError, }; use blockscout_db::entity::blocks; @@ -52,12 +52,12 @@ struct BlockTimestamp { async fn query_average_block_time( cx: &UpdateContext<'_>, offset: u64, -) -> Result>, UpdateError> { +) -> Result>, ChartError> { let query = average_block_time_statement(offset); let block_timestamps = BlockTimestamp::find_by_statement(query) .all(cx.blockscout.connection.as_ref()) .await - .map_err(UpdateError::BlockscoutDB)?; + .map_err(ChartError::BlockscoutDB)?; Ok(calculate_average_block_time(block_timestamps)) } @@ -69,12 +69,12 @@ impl RemoteQueryBehaviour for AverageBlockTimeQuery { async fn query_data( cx: &UpdateContext<'_>, _range: UniversalRange>, - ) -> Result, UpdateError> { + ) -> Result, ChartError> { match query_average_block_time(cx, OFFSET_BLOCKS).await? { Some(avg_block_time) => Ok(avg_block_time), None => query_average_block_time(cx, 0) .await? - .ok_or(UpdateError::Internal( + .ok_or(ChartError::Internal( "No blocks were returned to calculate average block time".into(), )), } diff --git a/stats/stats/src/charts/counters/mock.rs b/stats/stats/src/charts/counters/mock.rs index 0576f2443..701e2aace 100644 --- a/stats/stats/src/charts/counters/mock.rs +++ b/stats/stats/src/charts/counters/mock.rs @@ -11,7 +11,7 @@ use crate::{ }, range::UniversalRange, types::timespans::DateValue, - ChartProperties, Named, UpdateError, + ChartProperties, Named, ChartError, }; use chrono::{DateTime, NaiveDate, Utc}; @@ -32,7 +32,7 @@ where async fn query_data( cx: &UpdateContext<'_>, _range: UniversalRange>, - ) -> Result { + ) -> Result { if cx.time >= PointDateTime::get() { Ok(DateValue:: { timespan: PointDateTime::get().date_naive(), diff --git a/stats/stats/src/charts/counters/total_blocks.rs b/stats/stats/src/charts/counters/total_blocks.rs index 5e006e1d2..24cdc94a1 100644 --- a/stats/stats/src/charts/counters/total_blocks.rs +++ b/stats/stats/src/charts/counters/total_blocks.rs @@ -12,7 +12,7 @@ use crate::{ range::UniversalRange, types::timespans::DateValue, utils::MarkedDbConnection, - ChartProperties, MissingDatePolicy, Named, UpdateError, + ChartProperties, MissingDatePolicy, Named, ChartError, }; use blockscout_db::entity::blocks; @@ -36,7 +36,7 @@ impl RemoteQueryBehaviour for TotalBlocksQueryBehaviour { async fn query_data( cx: &UpdateContext<'_>, _range: UniversalRange>, - ) -> Result { + ) -> Result { let data = blocks::Entity::find() .select_only() .column_as(Expr::col(blocks::Column::Number).count(), "number") @@ -45,8 +45,8 @@ impl RemoteQueryBehaviour for TotalBlocksQueryBehaviour { .into_model::() .one(cx.blockscout.connection.as_ref()) .await - .map_err(UpdateError::BlockscoutDB)? - .ok_or_else(|| UpdateError::Internal("query returned nothing".into()))?; + .map_err(ChartError::BlockscoutDB)? + .ok_or_else(|| ChartError::Internal("query returned nothing".into()))?; let data = DateValue:: { timespan: data.timestamp.date(), @@ -85,7 +85,7 @@ pub static TOTAL_BLOCKS_ESTIMATION_CACHE_LIVENESS_SEC: OnceLock = OnceLock: static CACHED_BLOCKS_ESTIMATION: OnceLock>> = OnceLock::new(); impl ValueEstimation for CachedBlocksEstimation { - async fn estimate(blockscout: &MarkedDbConnection) -> Result, UpdateError> { + async fn estimate(blockscout: &MarkedDbConnection) -> Result, ChartError> { async fn cached_blocks_estimation( blockscout: &DatabaseConnection, db_id: &str, @@ -115,7 +115,7 @@ impl ValueEstimation for CachedBlocksEstimation { let now = Utc::now(); let value = cached_blocks_estimation(blockscout.connection.as_ref(), &blockscout.db_name) .await - .map_err(UpdateError::BlockscoutDB)? + .map_err(ChartError::BlockscoutDB)? .map(|b| { let b = b as f64 * 0.9; b as i64 diff --git a/stats/stats/src/charts/counters/total_contracts.rs b/stats/stats/src/charts/counters/total_contracts.rs index 30b342fd7..c5eeeab67 100644 --- a/stats/stats/src/charts/counters/total_contracts.rs +++ b/stats/stats/src/charts/counters/total_contracts.rs @@ -8,7 +8,7 @@ use crate::{ }, range::UniversalRange, types::timespans::DateValue, - ChartProperties, MissingDatePolicy, Named, UpdateError, + ChartProperties, MissingDatePolicy, Named, ChartError, }; use blockscout_db::entity::addresses; @@ -24,13 +24,13 @@ impl RemoteQueryBehaviour for TotalContractsQueryBehaviour { async fn query_data( cx: &UpdateContext<'_>, _range: UniversalRange>, - ) -> Result { + ) -> Result { let value = addresses::Entity::find() .filter(addresses::Column::ContractCode.is_not_null()) .filter(addresses::Column::InsertedAt.lte(cx.time)) .count(cx.blockscout.connection.as_ref()) .await - .map_err(UpdateError::BlockscoutDB)?; + .map_err(ChartError::BlockscoutDB)?; let timespan = cx.time.date_naive(); Ok(DateValue:: { timespan, diff --git a/stats/stats/src/charts/counters/yesterday_txns.rs b/stats/stats/src/charts/counters/yesterday_txns.rs index 4cfa9a31e..f16f39cc8 100644 --- a/stats/stats/src/charts/counters/yesterday_txns.rs +++ b/stats/stats/src/charts/counters/yesterday_txns.rs @@ -10,7 +10,7 @@ use crate::{ range::UniversalRange, types::TimespanValue, utils::day_start, - ChartProperties, MissingDatePolicy, Named, UpdateError, + ChartProperties, MissingDatePolicy, Named, ChartError, }; use chrono::{DateTime, Days, NaiveDate, Utc}; use entity::sea_orm_active_enums::ChartType; @@ -24,11 +24,11 @@ impl RemoteQueryBehaviour for YesterdayTxnsQuery { async fn query_data( cx: &UpdateContext<'_>, _range: UniversalRange>, - ) -> Result { + ) -> Result { let today = cx.time.date_naive(); let yesterday = today .checked_sub_days(Days::new(1)) - .ok_or(UpdateError::Internal( + .ok_or(ChartError::Internal( "Update time is incorrect: ~ minimum possible date".into(), ))?; let yesterday_range = day_start(&yesterday)..day_start(&today); @@ -39,7 +39,7 @@ impl RemoteQueryBehaviour for YesterdayTxnsQuery { let data = Self::Output::find_by_statement(query) .one(cx.blockscout.connection.as_ref()) .await - .map_err(UpdateError::BlockscoutDB)? + .map_err(ChartError::BlockscoutDB)? // no transactions for yesterday .unwrap_or(TimespanValue::with_zero_value(yesterday)); Ok(data) diff --git a/stats/stats/src/charts/db_interaction/read.rs b/stats/stats/src/charts/db_interaction/read.rs index 271ed4724..7b55f639a 100644 --- a/stats/stats/src/charts/db_interaction/read.rs +++ b/stats/stats/src/charts/db_interaction/read.rs @@ -10,7 +10,7 @@ use crate::{ timespans::{DateValue, Month, Week, Year}, ExtendedTimespanValue, Timespan, TimespanDuration, TimespanValue, }, - ChartProperties, MissingDatePolicy, UpdateError, + ChartProperties, MissingDatePolicy, ChartError, }; use blockscout_db::entity::blocks; @@ -482,7 +482,7 @@ pub async fn last_accurate_point( force_full: bool, approximate_trailing_points: u64, policy: MissingDatePolicy, -) -> Result>, UpdateError> +) -> Result>, ChartError> where ChartProps: ChartProperties + ?Sized, ChartProps::Resolution: Ord + Clone + Debug, @@ -499,7 +499,7 @@ where .into_model() .one(db) .await - .map_err(UpdateError::StatsDB)?; + .map_err(ChartError::StatsDB)?; let metadata = get_chart_metadata(db, &ChartProps::key()).await?; match recorded_min_blockscout_block { @@ -536,7 +536,7 @@ where } }); let Some(last_accurate_point) = last_accurate_point else { - return Err(UpdateError::Internal("Failure while reading chart data: did not return accurate data (with `fill_missing_dates`=true)".into())); + return Err(ChartError::Internal("Failure while reading chart data: did not return accurate data (with `fill_missing_dates`=true)".into())); }; if let Some(block) = recorded_min_blockscout_block.min_blockscout_block { @@ -656,10 +656,10 @@ impl RemoteQueryBehaviour for QueryAllBlockTimestampRange { async fn query_data( cx: &UpdateContext<'_>, _range: UniversalRange>, - ) -> Result { + ) -> Result { let start_timestamp = get_min_date_blockscout(cx.blockscout.connection.as_ref()) .await - .map_err(UpdateError::BlockscoutDB)? + .map_err(ChartError::BlockscoutDB)? .and_utc(); Ok(start_timestamp..cx.time) } diff --git a/stats/stats/src/charts/lines/gas_used_growth.rs b/stats/stats/src/charts/lines/gas_used_growth.rs index 6b9cc3bfe..cdce9900d 100644 --- a/stats/stats/src/charts/lines/gas_used_growth.rs +++ b/stats/stats/src/charts/lines/gas_used_growth.rs @@ -21,7 +21,7 @@ use crate::{ define_and_impl_resolution_properties, types::timespans::{DateValue, Month, Week, Year}, utils::sql_with_range_filter_opt, - ChartProperties, MissingDatePolicy, Named, UpdateError, + ChartError, ChartProperties, MissingDatePolicy, Named, }; use chrono::{DateTime, NaiveDate, Utc}; @@ -61,7 +61,7 @@ pub struct IncrementsFromPartialSum; impl MapFunction>> for IncrementsFromPartialSum { type Output = Vec>; - fn function(inner_data: Vec>) -> Result { + fn function(inner_data: Vec>) -> Result { Ok(inner_data .into_iter() .scan(Decimal::ZERO, |state, mut next| { diff --git a/stats/stats/src/charts/lines/mock.rs b/stats/stats/src/charts/lines/mock.rs index 2dcecb1c3..ac6b5a151 100644 --- a/stats/stats/src/charts/lines/mock.rs +++ b/stats/stats/src/charts/lines/mock.rs @@ -12,7 +12,7 @@ use crate::{ missing_date::fit_into_range, range::{Incrementable, UniversalRange}, types::{timespans::DateValue, Timespan, TimespanValue}, - ChartProperties, MissingDatePolicy, Named, UpdateError, + ChartProperties, MissingDatePolicy, Named, ChartError, }; use chrono::{DateTime, Duration, NaiveDate, Utc}; @@ -100,7 +100,7 @@ where async fn query_data( cx: &UpdateContext<'_>, range: UniversalRange>, - ) -> Result>, UpdateError> { + ) -> Result>, ChartError> { let full_data = mocked_lines(DateRange::get(), ValueRange::get()); Ok(mock_trim_lines(full_data, cx.time, range, Policy::get())) } @@ -146,7 +146,7 @@ where async fn query_data( cx: &UpdateContext<'_>, range: UniversalRange>, - ) -> Result { + ) -> Result { Ok(mock_trim_lines(Data::get(), cx.time, range, Policy::get())) } } diff --git a/stats/stats/src/charts/lines/native_coin_holders_growth.rs b/stats/stats/src/charts/lines/native_coin_holders_growth.rs index ef5536c89..f9b82e997 100644 --- a/stats/stats/src/charts/lines/native_coin_holders_growth.rs +++ b/stats/stats/src/charts/lines/native_coin_holders_growth.rs @@ -21,7 +21,7 @@ use crate::{ }, define_and_impl_resolution_properties, types::timespans::{DateValue, Month, Week, Year}, - ChartProperties, MissingDatePolicy, Named, UpdateError, + ChartProperties, MissingDatePolicy, Named, ChartError, }; use blockscout_db::entity::address_coin_balances_daily; @@ -96,7 +96,7 @@ impl UpdateBehaviour<(), (), NaiveDate> for Update { last_accurate_point: Option>, min_blockscout_block: i64, dependency_data_fetch_timer: &mut AggregateTimer, - ) -> Result<(), UpdateError> { + ) -> Result<(), ChartError> { update_sequentially_with_support_table( cx, chart_id, @@ -115,7 +115,7 @@ pub async fn update_sequentially_with_support_table( last_accurate_point: Option>, min_blockscout_block: i64, remote_fetch_timer: &mut AggregateTimer, -) -> Result<(), UpdateError> { +) -> Result<(), ChartError> { tracing::info!(chart =% Properties::key(), "start sequential update"); let all_days = match last_accurate_point { Some(last_row) => get_unique_ordered_days( @@ -124,14 +124,14 @@ pub async fn update_sequentially_with_support_table( remote_fetch_timer, ) .await - .map_err(UpdateError::BlockscoutDB)?, + .map_err(ChartError::BlockscoutDB)?, None => { clear_support_table(cx.db.connection.as_ref()) .await - .map_err(UpdateError::BlockscoutDB)?; + .map_err(ChartError::BlockscoutDB)?; get_unique_ordered_days(cx.blockscout.connection.as_ref(), None, remote_fetch_timer) .await - .map_err(UpdateError::BlockscoutDB)? + .map_err(ChartError::BlockscoutDB)? } }; @@ -151,21 +151,21 @@ pub async fn update_sequentially_with_support_table( .connection .begin() .await - .map_err(UpdateError::StatsDB)?; + .map_err(ChartError::StatsDB)?; let data: Vec = calculate_days_using_support_table( &db_tx, cx.blockscout.connection.as_ref(), days.iter().copied(), ) .await - .map_err(|e| UpdateError::Internal(e.to_string()))? + .map_err(|e| ChartError::Internal(e.to_string()))? .into_iter() .map(|result| result.active_model(chart_id, Some(min_blockscout_block))) .collect(); insert_data_many(&db_tx, data) .await - .map_err(UpdateError::StatsDB)?; - db_tx.commit().await.map_err(UpdateError::StatsDB)?; + .map_err(ChartError::StatsDB)?; + db_tx.commit().await.map_err(ChartError::StatsDB)?; } Ok(()) } @@ -174,7 +174,7 @@ async fn calculate_days_using_support_table( db: &C1, blockscout: &C2, days: impl IntoIterator, -) -> Result>, UpdateError> +) -> Result>, ChartError> where C1: ConnectionTrait, C2: ConnectionTrait, @@ -182,7 +182,7 @@ where let mut result = vec![]; let new_holders_by_date = get_holder_changes_by_date(blockscout, days) .await - .map_err(|e| UpdateError::Internal(format!("cannot get new holders: {e}")))?; + .map_err(|e| ChartError::Internal(format!("cannot get new holders: {e}")))?; for (date, holders) in new_holders_by_date { // this check shouldnt be triggered if data in blockscout is correct, @@ -190,7 +190,7 @@ where let addresses = holders.iter().map(|h| &h.address).collect::>(); if addresses.len() != holders.len() { tracing::error!(addresses = ?addresses, date = ?date, "duplicate addresses in holders"); - return Err(UpdateError::Internal( + return Err(ChartError::Internal( "duplicate addresses in holders".to_string(), )); }; @@ -203,10 +203,10 @@ where update_current_holders(db, holders) .await - .map_err(|e| UpdateError::Internal(format!("cannot update holders: {e}")))?; + .map_err(|e| ChartError::Internal(format!("cannot update holders: {e}")))?; let new_count = count_current_holders(db) .await - .map_err(|e| UpdateError::Internal(format!("cannot count holders: {e}")))?; + .map_err(|e| ChartError::Internal(format!("cannot count holders: {e}")))?; result.push(DateValue:: { timespan: date, value: new_count.to_string(), diff --git a/stats/stats/src/charts/lines/new_accounts.rs b/stats/stats/src/charts/lines/new_accounts.rs index 2c2f9f30f..24f5da7a6 100644 --- a/stats/stats/src/charts/lines/new_accounts.rs +++ b/stats/stats/src/charts/lines/new_accounts.rs @@ -24,7 +24,7 @@ use crate::{ range::{data_source_query_range_to_db_statement_range, UniversalRange}, types::timespans::{Month, Week, Year}, utils::sql_with_range_filter_opt, - ChartProperties, Named, UpdateError, + ChartError, ChartProperties, Named, }; use chrono::{DateTime, NaiveDate, Utc}; @@ -101,7 +101,7 @@ impl RemoteQueryBehaviour for NewAccountsQueryBehaviour { async fn query_data( cx: &UpdateContext<'_>, range: UniversalRange>, - ) -> Result>, UpdateError> { + ) -> Result>, ChartError> { let statement_range = data_source_query_range_to_db_statement_range::(cx, range) .await?; @@ -112,7 +112,7 @@ impl RemoteQueryBehaviour for NewAccountsQueryBehaviour { let mut data = DateValue::::find_by_statement(query) .all(cx.blockscout.connection.as_ref()) .await - .map_err(UpdateError::BlockscoutDB)?; + .map_err(ChartError::BlockscoutDB)?; // make sure that it's sorted data.sort_by_key(|d| d.timespan); if let Some(range) = statement_range { diff --git a/stats/stats/src/charts/mod.rs b/stats/stats/src/charts/mod.rs index c7abc1d27..510e7b7e9 100644 --- a/stats/stats/src/charts/mod.rs +++ b/stats/stats/src/charts/mod.rs @@ -6,5 +6,5 @@ pub mod query_dispatch; pub mod types; pub use chart::{ chart_properties_portrait, ChartKey, ChartObject, ChartProperties, ChartPropertiesObject, - MissingDatePolicy, Named, ResolutionKind, UpdateError, + MissingDatePolicy, Named, ResolutionKind, ChartError, }; diff --git a/stats/stats/src/charts/query_dispatch.rs b/stats/stats/src/charts/query_dispatch.rs index 169b67f6f..997bd7b12 100644 --- a/stats/stats/src/charts/query_dispatch.rs +++ b/stats/stats/src/charts/query_dispatch.rs @@ -18,7 +18,7 @@ use crate::{ use super::{ types::{ExtendedTimespanValue, Timespan, TimespanValue}, - ChartProperties, UpdateError, + ChartProperties, ChartError, }; /// Data query trait with unified data format (for external use) @@ -39,7 +39,7 @@ pub trait QuerySerialized { range: UniversalRange>, points_limit: Option, fill_missing_dates: bool, - ) -> Pin> + Send + 'a>>; + ) -> Pin> + Send + 'a>>; } /// [`QuerySerialized`] but for dynamic dispatch @@ -142,7 +142,7 @@ where range: UniversalRange>, points_limit: Option, fill_missing_dates: bool, - ) -> Pin> + Send + 'a>> + ) -> Pin> + Send + 'a>> { let cx = cx.clone(); Box::pin(async move { diff --git a/stats/stats/src/data_processing.rs b/stats/stats/src/data_processing.rs index 99195854a..73cc1e3d1 100644 --- a/stats/stats/src/data_processing.rs +++ b/stats/stats/src/data_processing.rs @@ -3,7 +3,7 @@ use chrono::NaiveDate; use crate::{ charts::types::timespans::DateValue, types::{Timespan, TimespanValue}, - UpdateError, + ChartError, }; use std::{ mem, @@ -17,7 +17,7 @@ use std::{ pub fn cumsum( mut data: Vec>, mut prev_sum: Value, -) -> Result>, UpdateError> +) -> Result>, ChartError> where Value: AddAssign + Clone, TimespanValue: Default, @@ -40,7 +40,7 @@ where pub fn deltas( mut data: Vec>, mut prev_value: Value, -) -> Result>, UpdateError> +) -> Result>, ChartError> where Value: SubAssign + Clone, TimespanValue: Default, @@ -61,7 +61,7 @@ where pub fn sum( data: &[TimespanValue], mut partial_sum: Value, -) -> Result, UpdateError> +) -> Result, ChartError> where Resolution: Timespan + Clone + Ord, Value: AddAssign + Clone, diff --git a/stats/stats/src/data_source/kinds/auxiliary/cumulative.rs b/stats/stats/src/data_source/kinds/auxiliary/cumulative.rs index 6fa509bc1..ca77c9c99 100644 --- a/stats/stats/src/data_source/kinds/auxiliary/cumulative.rs +++ b/stats/stats/src/data_source/kinds/auxiliary/cumulative.rs @@ -10,7 +10,7 @@ use crate::{ data_source::{DataSource, UpdateContext}, range::UniversalRange, types::TimespanValue, - UpdateError, + ChartError, }; /// Auxiliary source for cumulative chart. @@ -43,7 +43,7 @@ where Ok(()) } - async fn update_itself(_cx: &UpdateContext<'_>) -> Result<(), UpdateError> { + async fn update_itself(_cx: &UpdateContext<'_>) -> Result<(), ChartError> { // just an adapter; inner is handled recursively Ok(()) } @@ -52,7 +52,7 @@ where cx: &UpdateContext<'_>, range: UniversalRange>, dependency_data_fetch_timer: &mut AggregateTimer, - ) -> Result { + ) -> Result { let delta_data = Delta::query_data(cx, range, dependency_data_fetch_timer).await?; let data = cumsum::(delta_data, Value::zero())?; Ok(data) diff --git a/stats/stats/src/data_source/kinds/data_manipulation/delta.rs b/stats/stats/src/data_source/kinds/data_manipulation/delta.rs index d07762d70..9aee44706 100644 --- a/stats/stats/src/data_source/kinds/data_manipulation/delta.rs +++ b/stats/stats/src/data_source/kinds/data_manipulation/delta.rs @@ -15,7 +15,7 @@ use crate::{ data_source::{DataSource, UpdateContext}, range::UniversalRange, types::TimespanValue, - UpdateError, + ChartError, }; /// Calculate delta data from cumulative dependency. @@ -52,7 +52,7 @@ where Ok(()) } - async fn update_itself(_cx: &UpdateContext<'_>) -> Result<(), UpdateError> { + async fn update_itself(_cx: &UpdateContext<'_>) -> Result<(), ChartError> { // just an adapter; inner is handled recursively Ok(()) } @@ -61,7 +61,7 @@ where cx: &UpdateContext<'_>, range: UniversalRange>, dependency_data_fetch_timer: &mut AggregateTimer, - ) -> Result { + ) -> Result { let mut request_range = range.clone(); request_range.start = request_range.start.map(|s| { s.checked_sub_signed(TimeDelta::days(1)) diff --git a/stats/stats/src/data_source/kinds/data_manipulation/filter_deducible.rs b/stats/stats/src/data_source/kinds/data_manipulation/filter_deducible.rs index 73ce9c9ad..28728ffa4 100644 --- a/stats/stats/src/data_source/kinds/data_manipulation/filter_deducible.rs +++ b/stats/stats/src/data_source/kinds/data_manipulation/filter_deducible.rs @@ -12,7 +12,7 @@ use crate::{ data_source::{DataSource, UpdateContext}, range::UniversalRange, types::TimespanValue, - ChartProperties, MissingDatePolicy, UpdateError, + ChartProperties, MissingDatePolicy, ChartError, }; /// Pass only essential points from `D`, removing ones that can be deduced @@ -45,7 +45,7 @@ where Ok(()) } - async fn update_itself(_cx: &UpdateContext<'_>) -> Result<(), UpdateError> { + async fn update_itself(_cx: &UpdateContext<'_>) -> Result<(), ChartError> { // just an adapter; inner is handled recursively Ok(()) } @@ -54,7 +54,7 @@ where cx: &UpdateContext<'_>, range: UniversalRange>, dependency_data_fetch_timer: &mut AggregateTimer, - ) -> Result { + ) -> Result { let data = DS::query_data(cx, range, dependency_data_fetch_timer).await?; Ok(match Properties::missing_date_policy() { MissingDatePolicy::FillZero => { diff --git a/stats/stats/src/data_source/kinds/data_manipulation/last_point.rs b/stats/stats/src/data_source/kinds/data_manipulation/last_point.rs index e502f83d2..c3fcb11ad 100644 --- a/stats/stats/src/data_source/kinds/data_manipulation/last_point.rs +++ b/stats/stats/src/data_source/kinds/data_manipulation/last_point.rs @@ -13,7 +13,7 @@ use crate::{ range::UniversalRange, types::{Timespan, TimespanValue, ZeroTimespanValue}, utils::day_start, - UpdateError, + ChartError, }; pub struct LastPoint(PhantomData) @@ -42,7 +42,7 @@ where Ok(()) } - async fn update_itself(_cx: &UpdateContext<'_>) -> Result<(), UpdateError> { + async fn update_itself(_cx: &UpdateContext<'_>) -> Result<(), ChartError> { // just an adapter; inner is handled recursively Ok(()) } @@ -51,7 +51,7 @@ where cx: &UpdateContext<'_>, _range: UniversalRange>, dependency_data_fetch_timer: &mut AggregateTimer, - ) -> Result { + ) -> Result { let data = DS::query_data( cx, (day_start(&cx.time.date_naive())..cx.time).into(), diff --git a/stats/stats/src/data_source/kinds/data_manipulation/map/mod.rs b/stats/stats/src/data_source/kinds/data_manipulation/map/mod.rs index a4449250d..a0d114b64 100644 --- a/stats/stats/src/data_source/kinds/data_manipulation/map/mod.rs +++ b/stats/stats/src/data_source/kinds/data_manipulation/map/mod.rs @@ -10,7 +10,7 @@ use sea_orm::{DatabaseConnection, DbErr}; use crate::{ data_source::{DataSource, UpdateContext}, range::UniversalRange, - UpdateError, + ChartError, }; mod parse; @@ -29,7 +29,7 @@ where pub trait MapFunction { type Output: Send; - fn function(inner_data: Input) -> Result; + fn function(inner_data: Input) -> Result; } impl DataSource for Map @@ -52,7 +52,7 @@ where Ok(()) } - async fn update_itself(_cx: &UpdateContext<'_>) -> Result<(), UpdateError> { + async fn update_itself(_cx: &UpdateContext<'_>) -> Result<(), ChartError> { // just an adapter; inner is handled recursively Ok(()) } @@ -61,7 +61,7 @@ where cx: &UpdateContext<'_>, range: UniversalRange>, dependency_data_fetch_timer: &mut AggregateTimer, - ) -> Result { + ) -> Result { let inner_data = ::query_data(cx, range, dependency_data_fetch_timer).await?; F::function(inner_data) diff --git a/stats/stats/src/data_source/kinds/data_manipulation/map/parse.rs b/stats/stats/src/data_source/kinds/data_manipulation/map/parse.rs index bac618b32..c1a9b88ca 100644 --- a/stats/stats/src/data_source/kinds/data_manipulation/map/parse.rs +++ b/stats/stats/src/data_source/kinds/data_manipulation/map/parse.rs @@ -1,7 +1,7 @@ use std::{fmt::Display, marker::PhantomData, str::FromStr}; use crate::{ - data_source::kinds::data_manipulation::map::MapFunction, types::TimespanValue, UpdateError, + data_source::kinds::data_manipulation::map::MapFunction, types::TimespanValue, ChartError, }; use super::Map; @@ -19,19 +19,19 @@ where fn function( inner_data: Vec>, - ) -> Result>, UpdateError> { + ) -> Result>, ChartError> { inner_data .into_iter() .map(|p| { let val_parsed = p.value.parse::().map_err(|e| { - UpdateError::Internal(format!("failed to parse values of dependency: {e}")) + ChartError::Internal(format!("failed to parse values of dependency: {e}")) })?; Ok(TimespanValue { timespan: p.timespan, value: val_parsed, }) }) - .collect::, UpdateError>>() + .collect::, ChartError>>() } } @@ -45,9 +45,9 @@ where fn function( inner_data: TimespanValue, - ) -> Result { + ) -> Result { let val_parsed = inner_data.value.parse::().map_err(|e| { - UpdateError::Internal(format!("failed to parse values of dependency: {e}")) + ChartError::Internal(format!("failed to parse values of dependency: {e}")) })?; Ok(TimespanValue { timespan: inner_data.timespan, diff --git a/stats/stats/src/data_source/kinds/data_manipulation/map/strip_extension.rs b/stats/stats/src/data_source/kinds/data_manipulation/map/strip_extension.rs index 80de37e0f..f0accab13 100644 --- a/stats/stats/src/data_source/kinds/data_manipulation/map/strip_extension.rs +++ b/stats/stats/src/data_source/kinds/data_manipulation/map/strip_extension.rs @@ -1,6 +1,6 @@ use crate::{ types::{ExtendedTimespanValue, TimespanValue}, - UpdateError, + ChartError, }; use super::{Map, MapFunction}; @@ -16,7 +16,7 @@ where V: Send, { type Output = Vec>; - fn function(inner_data: Vec>) -> Result { + fn function(inner_data: Vec>) -> Result { Ok(inner_data.into_iter().map(|p| p.into()).collect()) } } @@ -27,7 +27,7 @@ where V: Send, { type Output = TimespanValue; - fn function(inner_data: ExtendedTimespanValue) -> Result { + fn function(inner_data: ExtendedTimespanValue) -> Result { Ok(inner_data.into()) } } diff --git a/stats/stats/src/data_source/kinds/data_manipulation/map/to_string.rs b/stats/stats/src/data_source/kinds/data_manipulation/map/to_string.rs index ccf6a3d11..bf6b2dc98 100644 --- a/stats/stats/src/data_source/kinds/data_manipulation/map/to_string.rs +++ b/stats/stats/src/data_source/kinds/data_manipulation/map/to_string.rs @@ -1,4 +1,4 @@ -use crate::{types::TimespanValue, UpdateError}; +use crate::{types::TimespanValue, ChartError}; use super::{Map, MapFunction}; @@ -12,7 +12,7 @@ where type Output = Vec>; fn function( inner_data: Vec>, - ) -> Result { + ) -> Result { Ok(inner_data.into_iter().map(|p| p.into()).collect()) } } @@ -23,7 +23,7 @@ where TimespanValue: Into>, { type Output = TimespanValue; - fn function(inner_data: TimespanValue) -> Result { + fn function(inner_data: TimespanValue) -> Result { Ok(inner_data.into()) } } diff --git a/stats/stats/src/data_source/kinds/data_manipulation/resolutions/average.rs b/stats/stats/src/data_source/kinds/data_manipulation/resolutions/average.rs index 0560c591e..1b96272c6 100644 --- a/stats/stats/src/data_source/kinds/data_manipulation/resolutions/average.rs +++ b/stats/stats/src/data_source/kinds/data_manipulation/resolutions/average.rs @@ -12,7 +12,7 @@ use crate::{ }, range::UniversalRange, types::{ConsistsOf, Timespan, TimespanValue}, - UpdateError, + ChartError, }; use super::extend_to_timespan_boundaries; @@ -55,7 +55,7 @@ where Ok(()) } - async fn update_itself(_cx: &UpdateContext<'_>) -> Result<(), UpdateError> { + async fn update_itself(_cx: &UpdateContext<'_>) -> Result<(), ChartError> { // just an adapter; inner is handled recursively Ok(()) } @@ -64,7 +64,7 @@ where cx: &UpdateContext<'_>, range: UniversalRange>, dependency_data_fetch_timer: &mut AggregateTimer, - ) -> Result { + ) -> Result { let time_range_for_lower_res = extend_to_timespan_boundaries::(range); let high_res_averages = Average::query_data( cx, diff --git a/stats/stats/src/data_source/kinds/data_manipulation/resolutions/last_value.rs b/stats/stats/src/data_source/kinds/data_manipulation/resolutions/last_value.rs index 1789384a8..8e14e9c74 100644 --- a/stats/stats/src/data_source/kinds/data_manipulation/resolutions/last_value.rs +++ b/stats/stats/src/data_source/kinds/data_manipulation/resolutions/last_value.rs @@ -13,7 +13,7 @@ use crate::{ data_source::{DataSource, UpdateContext}, range::UniversalRange, types::{ConsistsOf, Timespan, TimespanValue}, - UpdateError, + ChartError, }; use super::{extend_to_timespan_boundaries, reduce_each_timespan}; @@ -45,7 +45,7 @@ where Ok(()) } - async fn update_itself(_cx: &UpdateContext<'_>) -> Result<(), UpdateError> { + async fn update_itself(_cx: &UpdateContext<'_>) -> Result<(), ChartError> { // just an adapter; inner is handled recursively Ok(()) } @@ -54,7 +54,7 @@ where cx: &UpdateContext<'_>, range: UniversalRange>, dependency_data_fetch_timer: &mut AggregateTimer, - ) -> Result { + ) -> Result { let time_range_for_lower_res = extend_to_timespan_boundaries::(range); let high_res_data = DS::query_data(cx, time_range_for_lower_res, dependency_data_fetch_timer).await?; diff --git a/stats/stats/src/data_source/kinds/data_manipulation/resolutions/sum.rs b/stats/stats/src/data_source/kinds/data_manipulation/resolutions/sum.rs index 24bd2ed27..c6939d1d6 100644 --- a/stats/stats/src/data_source/kinds/data_manipulation/resolutions/sum.rs +++ b/stats/stats/src/data_source/kinds/data_manipulation/resolutions/sum.rs @@ -14,7 +14,7 @@ use crate::{ data_source::{DataSource, UpdateContext}, range::UniversalRange, types::{ConsistsOf, Timespan, TimespanValue}, - UpdateError, + ChartError, }; use super::{extend_to_timespan_boundaries, reduce_each_timespan}; @@ -46,7 +46,7 @@ where Ok(()) } - async fn update_itself(_cx: &UpdateContext<'_>) -> Result<(), UpdateError> { + async fn update_itself(_cx: &UpdateContext<'_>) -> Result<(), ChartError> { // just an adapter; inner is handled recursively Ok(()) } @@ -55,7 +55,7 @@ where cx: &UpdateContext<'_>, range: UniversalRange>, dependency_data_fetch_timer: &mut AggregateTimer, - ) -> Result { + ) -> Result { let time_range_for_lower_res = extend_to_timespan_boundaries::(range); let high_res_data = DS::query_data(cx, time_range_for_lower_res, dependency_data_fetch_timer).await?; diff --git a/stats/stats/src/data_source/kinds/data_manipulation/sum_point.rs b/stats/stats/src/data_source/kinds/data_manipulation/sum_point.rs index b8de19092..ba40c12f0 100644 --- a/stats/stats/src/data_source/kinds/data_manipulation/sum_point.rs +++ b/stats/stats/src/data_source/kinds/data_manipulation/sum_point.rs @@ -14,7 +14,7 @@ use crate::{ data_source::{source::DataSource, UpdateContext}, range::UniversalRange, types::{Timespan, TimespanValue}, - UpdateError, + ChartError, }; /// Sum all dependency's data. @@ -49,7 +49,7 @@ where Ok(()) } - async fn update_itself(_cx: &UpdateContext<'_>) -> Result<(), UpdateError> { + async fn update_itself(_cx: &UpdateContext<'_>) -> Result<(), ChartError> { // just an adapter; inner is handled recursively Ok(()) } @@ -58,7 +58,7 @@ where cx: &UpdateContext<'_>, _range: UniversalRange>, dependency_data_fetch_timer: &mut AggregateTimer, - ) -> Result { + ) -> Result { // it's possible to not request full data range and use last accurate point; // can be updated to work similarly to cumulative let full_data = diff --git a/stats/stats/src/data_source/kinds/local_db/mod.rs b/stats/stats/src/data_source/kinds/local_db/mod.rs index e73437f7f..2cabd4282 100644 --- a/stats/stats/src/data_source/kinds/local_db/mod.rs +++ b/stats/stats/src/data_source/kinds/local_db/mod.rs @@ -35,7 +35,7 @@ use crate::{ data_source::{DataSource, UpdateContext}, metrics, range::UniversalRange, - UpdateError, + ChartError, }; use super::auxiliary::PartialCumulative; @@ -141,7 +141,7 @@ where async fn update_itself_inner( cx: &UpdateContext<'_>, dependency_data_fetch_timer: &mut AggregateTimer, - ) -> Result<(), UpdateError> { + ) -> Result<(), ChartError> { let metadata = get_chart_metadata(cx.db.connection.as_ref(), &ChartProps::key()).await?; if let Some(last_updated_at) = metadata.last_updated_at { if postgres_timestamps_eq(cx.time, last_updated_at) { @@ -165,7 +165,7 @@ where let chart_id = metadata.id; let min_blockscout_block = get_min_block_blockscout(cx.blockscout.connection.as_ref()) .await - .map_err(UpdateError::BlockscoutDB)?; + .map_err(ChartError::BlockscoutDB)?; let last_accurate_point = last_accurate_point::( chart_id, min_blockscout_block, @@ -229,7 +229,7 @@ where Create::create(db, init_time).await } - async fn update_itself(cx: &UpdateContext<'_>) -> Result<(), UpdateError> { + async fn update_itself(cx: &UpdateContext<'_>) -> Result<(), ChartError> { // set up metrics + write some logs let mut dependency_data_fetch_timer = AggregateTimer::new(); @@ -260,7 +260,7 @@ where cx: &UpdateContext<'_>, range: UniversalRange>, dependency_data_fetch_timer: &mut AggregateTimer, - ) -> Result { + ) -> Result { let _timer = dependency_data_fetch_timer.start_interval(); // maybe add `fill_missing_dates` parameter to current function as well in the future // to get rid of "Note" in the `DataSource`'s method documentation @@ -329,7 +329,7 @@ mod tests { tests::{init_db::init_marked_db_all, mock_blockscout::fill_mock_blockscout_data}, types::{timespans::DateValue, TimespanValue}, update_group::{SyncUpdateGroup, UpdateGroup}, - ChartProperties, Named, UpdateError, + ChartProperties, Named, ChartError, }; type WasTriggeredStorage = Arc>; @@ -368,7 +368,7 @@ mod tests { _last_accurate_point: Option>, min_blockscout_block: i64, _dependency_data_fetch_timer: &mut AggregateTimer, - ) -> Result<(), UpdateError> { + ) -> Result<(), ChartError> { Self::record_trigger().await; // insert smth for dependency to work well let data = DateValue:: { @@ -378,7 +378,7 @@ mod tests { let value = data.active_model(chart_id, Some(min_blockscout_block)); insert_data_many(cx.db.connection.as_ref(), vec![value]) .await - .map_err(UpdateError::StatsDB)?; + .map_err(ChartError::StatsDB)?; Ok(()) } } diff --git a/stats/stats/src/data_source/kinds/local_db/parameter_traits.rs b/stats/stats/src/data_source/kinds/local_db/parameter_traits.rs index 4f3b88bf0..69a62a545 100644 --- a/stats/stats/src/data_source/kinds/local_db/parameter_traits.rs +++ b/stats/stats/src/data_source/kinds/local_db/parameter_traits.rs @@ -9,7 +9,7 @@ use crate::{ data_source::{DataSource, UpdateContext}, range::UniversalRange, types::TimespanValue, - RequestedPointsLimit, UpdateError, + RequestedPointsLimit, ChartError, }; /// In most cases, [`super::DefaultCreate`] is enough. @@ -38,18 +38,18 @@ where last_accurate_point: Option>, min_blockscout_block: i64, dependency_data_fetch_timer: &mut AggregateTimer, - ) -> impl Future> + Send; + ) -> impl Future> + Send; /// Update only chart metadata. fn update_metadata( db: &DatabaseConnection, chart_id: i32, update_time: DateTime, - ) -> impl Future> + Send { + ) -> impl Future> + Send { async move { set_last_updated_at(chart_id, db, update_time) .await - .map_err(UpdateError::StatsDB) + .map_err(ChartError::StatsDB) } } } @@ -65,5 +65,5 @@ pub trait QueryBehaviour { range: UniversalRange>, points_limit: Option, fill_missing_dates: bool, - ) -> impl Future> + Send; + ) -> impl Future> + Send; } diff --git a/stats/stats/src/data_source/kinds/local_db/parameters/query.rs b/stats/stats/src/data_source/kinds/local_db/parameters/query.rs index 476be1856..85268f1f7 100644 --- a/stats/stats/src/data_source/kinds/local_db/parameters/query.rs +++ b/stats/stats/src/data_source/kinds/local_db/parameters/query.rs @@ -9,7 +9,7 @@ use crate::{ range::UniversalRange, types::{timespans::DateValue, ExtendedTimespanValue, Timespan}, utils::MarkedDbConnection, - ChartProperties, RequestedPointsLimit, UpdateError, + ChartProperties, RequestedPointsLimit, ChartError, }; /// Usually the choice for line charts @@ -32,7 +32,7 @@ where range: UniversalRange>, points_limit: Option, fill_missing_dates: bool, - ) -> Result { + ) -> Result { // In DB we store data with date precision. Also, `get_line_chart_data` // works with inclusive range. Therefore, we need to convert the range and // get date without time. @@ -74,7 +74,7 @@ impl QueryBehaviour for DefaultQueryLast { _range: UniversalRange>, _points_limit: Option, _fill_missing_dates: bool, - ) -> Result { + ) -> Result { let value = get_counter_data( cx.db.connection.as_ref(), &C::name(), @@ -82,7 +82,7 @@ impl QueryBehaviour for DefaultQueryLast { C::missing_date_policy(), ) .await? - .ok_or(UpdateError::Internal(format!( + .ok_or(ChartError::Internal(format!( "no data for counter '{}' was found", C::name() )))?; @@ -92,7 +92,7 @@ impl QueryBehaviour for DefaultQueryLast { #[trait_variant::make(Send)] pub trait ValueEstimation { - async fn estimate(blockscout: &MarkedDbConnection) -> Result, UpdateError>; + async fn estimate(blockscout: &MarkedDbConnection) -> Result, ChartError>; } pub struct QueryLastWithEstimationFallback(PhantomData<(E, C)>) @@ -112,7 +112,7 @@ where _range: UniversalRange>, _points_limit: Option, _fill_missing_dates: bool, - ) -> Result { + ) -> Result { let value = match get_counter_data( cx.db.connection.as_ref(), &C::name(), @@ -142,7 +142,7 @@ mod tests { data_source::{types::BlockscoutMigrations, UpdateContext, UpdateParameters}, tests::init_db::init_marked_db_all, types::timespans::DateValue, - MissingDatePolicy, Named, UpdateError, + MissingDatePolicy, Named, ChartError, }; #[tokio::test] @@ -173,7 +173,7 @@ mod tests { impl ValueEstimation for TestFallback { async fn estimate( _blockscout: &MarkedDbConnection, - ) -> Result, UpdateError> { + ) -> Result, ChartError> { Ok(expected_estimate()) } } diff --git a/stats/stats/src/data_source/kinds/local_db/parameters/update/batching/mod.rs b/stats/stats/src/data_source/kinds/local_db/parameters/update/batching/mod.rs index 13ac7c346..9b1d9ab0d 100644 --- a/stats/stats/src/data_source/kinds/local_db/parameters/update/batching/mod.rs +++ b/stats/stats/src/data_source/kinds/local_db/parameters/update/batching/mod.rs @@ -19,7 +19,7 @@ use crate::{ }, range::UniversalRange, types::{ExtendedTimespanValue, Timespan, TimespanDuration, TimespanValue}, - ChartProperties, UpdateError, + ChartProperties, ChartError, }; pub mod parameter_traits; @@ -61,7 +61,7 @@ where last_accurate_point: Option>, min_blockscout_block: i64, dependency_data_fetch_timer: &mut AggregateTimer, - ) -> Result<(), UpdateError> { + ) -> Result<(), ChartError> { let now = cx.time; let update_from = last_accurate_point .clone() @@ -72,7 +72,7 @@ where get_min_date_blockscout(cx.blockscout.connection.as_ref()) .await .map(|time| time.date()) - .map_err(UpdateError::BlockscoutDB)?, + .map_err(ChartError::BlockscoutDB)?, ), }; @@ -129,7 +129,7 @@ where async fn get_previous_step_last_point( cx: &UpdateContext<'_>, this_step_start: Resolution, -) -> Result, UpdateError> +) -> Result, ChartError> where Resolution: Timespan + Clone, Query: QueryBehaviour>>, @@ -158,7 +158,7 @@ where )); if last_point_range_values.len() > 1 { // return error because it's likely that date in `previous_step_last_point` is incorrect - return Err(UpdateError::Internal("Retrieved 2 points from previous step; probably an issue with range construction and handling".to_owned())); + return Err(ChartError::Internal("Retrieved 2 points from previous step; probably an issue with range construction and handling".to_owned())); } Ok(previous_step_last_point) } @@ -171,7 +171,7 @@ async fn batch_update_values_step last_accurate_point: TimespanValue, range: BatchRange, dependency_data_fetch_timer: &mut AggregateTimer, -) -> Result +) -> Result where MainDep: DataSource, ResolutionDep: DataSource, @@ -235,12 +235,12 @@ fn generate_batch_ranges( start: Resolution, end: DateTime, max_step: TimespanDuration, -) -> Result>, UpdateError> +) -> Result>, ChartError> where Resolution: Timespan + Ord + Clone, { if max_step.repeats() == 0 { - return Err(UpdateError::Internal( + return Err(ChartError::Internal( "Zero maximum batch step is not allowed".into(), )); } diff --git a/stats/stats/src/data_source/kinds/local_db/parameters/update/batching/parameter_traits.rs b/stats/stats/src/data_source/kinds/local_db/parameters/update/batching/parameter_traits.rs index d2277ccb5..d4150c3c9 100644 --- a/stats/stats/src/data_source/kinds/local_db/parameters/update/batching/parameter_traits.rs +++ b/stats/stats/src/data_source/kinds/local_db/parameters/update/batching/parameter_traits.rs @@ -5,7 +5,7 @@ use sea_orm::DatabaseConnection; use crate::{ types::{Timespan, TimespanValue}, - UpdateError, + ChartError, }; pub trait BatchStepBehaviour @@ -25,5 +25,5 @@ where last_accurate_point: TimespanValue, main_data: MainInput, resolution_data: ResolutionInput, - ) -> impl Future> + std::marker::Send; + ) -> impl Future> + std::marker::Send; } diff --git a/stats/stats/src/data_source/kinds/local_db/parameters/update/batching/parameters/cumulative.rs b/stats/stats/src/data_source/kinds/local_db/parameters/update/batching/parameters/cumulative.rs index f8242e729..c01894e68 100644 --- a/stats/stats/src/data_source/kinds/local_db/parameters/update/batching/parameters/cumulative.rs +++ b/stats/stats/src/data_source/kinds/local_db/parameters/update/batching/parameters/cumulative.rs @@ -9,7 +9,7 @@ use sea_orm::DatabaseConnection; use crate::{ data_source::kinds::local_db::parameters::update::batching::parameter_traits::BatchStepBehaviour, types::{Timespan, TimespanValue}, - ChartProperties, UpdateError, + ChartProperties, ChartError, }; use super::PassVecStep; @@ -37,9 +37,9 @@ where last_accurate_point: TimespanValue, main_data: Vec>, _resolution_data: (), - ) -> Result { + ) -> Result { let partial_sum = last_accurate_point.value.parse::().map_err(|e| { - UpdateError::Internal(format!( + ChartError::Internal(format!( "failed to parse value in chart '{}': {e}", ChartProps::key() )) diff --git a/stats/stats/src/data_source/kinds/local_db/parameters/update/batching/parameters/mock.rs b/stats/stats/src/data_source/kinds/local_db/parameters/update/batching/parameters/mock.rs index 846acfd52..0d0cc61ec 100644 --- a/stats/stats/src/data_source/kinds/local_db/parameters/update/batching/parameters/mock.rs +++ b/stats/stats/src/data_source/kinds/local_db/parameters/update/batching/parameters/mock.rs @@ -5,7 +5,7 @@ use sea_orm::DatabaseConnection; use crate::{ data_source::kinds::local_db::parameters::update::batching::parameter_traits::BatchStepBehaviour, - tests::recorder::Recorder, types::timespans::DateValue, UpdateError, + tests::recorder::Recorder, types::timespans::DateValue, ChartError, }; use super::PassVecStep; @@ -40,7 +40,7 @@ where last_accurate_point: DateValue, main_data: Vec>, resolution_data: (), - ) -> Result { + ) -> Result { StepsRecorder::record(StepInput { chart_id, update_time, diff --git a/stats/stats/src/data_source/kinds/local_db/parameters/update/batching/parameters/mod.rs b/stats/stats/src/data_source/kinds/local_db/parameters/update/batching/parameters/mod.rs index ce27c16d3..41e274fd8 100644 --- a/stats/stats/src/data_source/kinds/local_db/parameters/update/batching/parameters/mod.rs +++ b/stats/stats/src/data_source/kinds/local_db/parameters/update/batching/parameters/mod.rs @@ -8,7 +8,7 @@ use crate::{ timespans::{Month, Week, Year}, Timespan, TimespanDuration, TimespanValue, }, - UpdateError, + ChartError, }; use super::parameter_traits::BatchStepBehaviour; @@ -42,7 +42,7 @@ where _last_accurate_point: TimespanValue, main_data: Vec>, _resolution_data: (), - ) -> Result { + ) -> Result { let found = main_data.len(); // note: right away cloning another chart will not result in exact copy, // because if the other chart is `FillPrevious`, then omitted starting point @@ -56,7 +56,7 @@ where .map(|value| value.active_model(chart_id, Some(min_blockscout_block))); insert_data_many(db, values) .await - .map_err(UpdateError::StatsDB)?; + .map_err(ChartError::StatsDB)?; Ok(found) } } diff --git a/stats/stats/src/data_source/kinds/local_db/parameters/update/point.rs b/stats/stats/src/data_source/kinds/local_db/parameters/update/point.rs index 774bc348c..f3b3c4a16 100644 --- a/stats/stats/src/data_source/kinds/local_db/parameters/update/point.rs +++ b/stats/stats/src/data_source/kinds/local_db/parameters/update/point.rs @@ -7,7 +7,7 @@ use crate::{ data_source::{kinds::local_db::UpdateBehaviour, DataSource, UpdateContext}, range::UniversalRange, types::{Timespan, TimespanValue}, - UpdateError, + ChartError, }; /// Store output of the `MainDep` right in the local db @@ -24,13 +24,13 @@ where _last_accurate_point: Option>, min_blockscout_block: i64, remote_fetch_timer: &mut AggregateTimer, - ) -> Result<(), UpdateError> { + ) -> Result<(), ChartError> { // range doesn't make sense there; thus is not used let data = MainDep::query_data(cx, UniversalRange::full(), remote_fetch_timer).await?; let value = data.active_model(chart_id, Some(min_blockscout_block)); insert_data_many(cx.db.connection.as_ref(), vec![value]) .await - .map_err(UpdateError::StatsDB)?; + .map_err(ChartError::StatsDB)?; Ok(()) } } diff --git a/stats/stats/src/data_source/kinds/remote_db/mod.rs b/stats/stats/src/data_source/kinds/remote_db/mod.rs index beb69e8d5..5a47180d4 100644 --- a/stats/stats/src/data_source/kinds/remote_db/mod.rs +++ b/stats/stats/src/data_source/kinds/remote_db/mod.rs @@ -30,7 +30,7 @@ use sea_orm::{DatabaseConnection, DbErr}; use crate::{ data_source::{source::DataSource, types::UpdateContext}, range::UniversalRange, - UpdateError, + ChartError, }; pub use query::{ @@ -48,7 +48,7 @@ pub trait RemoteQueryBehaviour { fn query_data( cx: &UpdateContext<'_>, range: UniversalRange>, - ) -> impl Future> + Send; + ) -> impl Future> + Send; } impl DataSource for RemoteDatabaseSource { @@ -71,12 +71,12 @@ impl DataSource for RemoteDatabaseSource { cx: &UpdateContext<'_>, range: UniversalRange>, remote_fetch_timer: &mut AggregateTimer, - ) -> Result<::Output, UpdateError> { + ) -> Result<::Output, ChartError> { let _interval = remote_fetch_timer.start_interval(); Q::query_data(cx, range).await } - async fn update_itself(_cx: &UpdateContext<'_>) -> Result<(), UpdateError> { + async fn update_itself(_cx: &UpdateContext<'_>) -> Result<(), ChartError> { Ok(()) } } diff --git a/stats/stats/src/data_source/kinds/remote_db/query/all.rs b/stats/stats/src/data_source/kinds/remote_db/query/all.rs index acdf2436d..0583240e4 100644 --- a/stats/stats/src/data_source/kinds/remote_db/query/all.rs +++ b/stats/stats/src/data_source/kinds/remote_db/query/all.rs @@ -13,7 +13,7 @@ use crate::{ }, range::{data_source_query_range_to_db_statement_range, UniversalRange}, types::TimespanValue, - UpdateError, + ChartError, }; pub trait StatementFromRange { @@ -54,7 +54,7 @@ where async fn query_data( cx: &UpdateContext<'_>, range: UniversalRange>, - ) -> Result>, UpdateError> { + ) -> Result>, ChartError> { // to not overcomplicate the queries let query_range = data_source_query_range_to_db_statement_range::(cx, range).await?; @@ -62,7 +62,7 @@ where let mut data = TimespanValue::::find_by_statement(query) .all(cx.blockscout.connection.as_ref()) .await - .map_err(UpdateError::BlockscoutDB)?; + .map_err(ChartError::BlockscoutDB)?; // linear time for sorted sequences data.sort_unstable_by(|a, b| a.timespan.cmp(&b.timespan)); // can't use sort_*_by_key: https://github.com/rust-lang/rust/issues/34162 diff --git a/stats/stats/src/data_source/kinds/remote_db/query/each.rs b/stats/stats/src/data_source/kinds/remote_db/query/each.rs index 13333b9eb..ea8dcf397 100644 --- a/stats/stats/src/data_source/kinds/remote_db/query/each.rs +++ b/stats/stats/src/data_source/kinds/remote_db/query/each.rs @@ -14,7 +14,7 @@ use crate::{ }, range::{exclusive_range_to_inclusive, UniversalRange}, types::{Timespan, TimespanValue}, - UpdateError, + ChartError, }; pub trait StatementFromTimespan { @@ -51,7 +51,7 @@ where async fn query_data( cx: &UpdateContext<'_>, range: UniversalRange>, - ) -> Result>, UpdateError> { + ) -> Result>, ChartError> { let query_range = if let Some(r) = range.clone().try_into_exclusive() { r } else { @@ -65,7 +65,7 @@ where let point_value = ValueWrapper::::find_by_statement(query) .one(cx.blockscout.connection.as_ref()) .await - .map_err(UpdateError::BlockscoutDB)?; + .map_err(ChartError::BlockscoutDB)?; if let Some(ValueWrapper { value }) = point_value { let timespan = resolution_from_range(point_range); collected_data.push(TimespanValue { timespan, value }); diff --git a/stats/stats/src/data_source/kinds/remote_db/query/one.rs b/stats/stats/src/data_source/kinds/remote_db/query/one.rs index aca5d1a68..1a1dcedc2 100644 --- a/stats/stats/src/data_source/kinds/remote_db/query/one.rs +++ b/stats/stats/src/data_source/kinds/remote_db/query/one.rs @@ -10,7 +10,7 @@ use crate::{ }, range::UniversalRange, types::TimespanValue, - UpdateError, + ChartError, }; pub trait StatementForOne { @@ -42,13 +42,13 @@ where async fn query_data( cx: &UpdateContext<'_>, _range: UniversalRange>, - ) -> Result, UpdateError> { + ) -> Result, ChartError> { let query = S::get_statement(&cx.blockscout_applied_migrations); let data = TimespanValue::::find_by_statement(query) .one(cx.blockscout.connection.as_ref()) .await - .map_err(UpdateError::BlockscoutDB)? - .ok_or_else(|| UpdateError::Internal("query returned nothing".into()))?; + .map_err(ChartError::BlockscoutDB)? + .ok_or_else(|| ChartError::Internal("query returned nothing".into()))?; Ok(data) } } diff --git a/stats/stats/src/data_source/source.rs b/stats/stats/src/data_source/source.rs index 4c72e4760..46507c9cf 100644 --- a/stats/stats/src/data_source/source.rs +++ b/stats/stats/src/data_source/source.rs @@ -7,7 +7,7 @@ use sea_orm::{DatabaseConnection, DbErr}; use tracing::instrument; use tynm::type_name; -use crate::{range::UniversalRange, UpdateError}; +use crate::{range::UniversalRange, ChartError}; use super::types::UpdateContext; @@ -116,7 +116,7 @@ pub trait DataSource { #[instrument(skip_all, level = tracing::Level::DEBUG, fields(source_mutex_id = Self::mutex_id()))] fn update_recursively( cx: &UpdateContext<'_>, - ) -> impl Future> + Send { + ) -> impl Future> + Send { async move { // Couldn't figure out how to control level per-argument basis (in instrumentation) // so this event is used insted, since the name is usually quite verbose @@ -146,9 +146,8 @@ pub trait DataSource { /// /// ## Description /// Update only thise data source's data (values + metadat) - fn update_itself( - cx: &UpdateContext<'_>, - ) -> impl Future> + Send; + fn update_itself(cx: &UpdateContext<'_>) + -> impl Future> + Send; /// Retrieve chart data. /// If `range` is `Some`, should return data within the range. Otherwise - all data. @@ -165,7 +164,7 @@ pub trait DataSource { cx: &UpdateContext<'_>, range: UniversalRange>, dependency_data_fetch_timer: &mut AggregateTimer, - ) -> impl Future> + Send; + ) -> impl Future> + Send; } // Base case for recursive type @@ -197,12 +196,12 @@ impl DataSource for () { HashSet::new() } - async fn update_recursively(_cx: &UpdateContext<'_>) -> Result<(), UpdateError> { + async fn update_recursively(_cx: &UpdateContext<'_>) -> Result<(), ChartError> { // stop recursion Ok(()) } - async fn update_itself(_cx: &UpdateContext<'_>) -> Result<(), UpdateError> { + async fn update_itself(_cx: &UpdateContext<'_>) -> Result<(), ChartError> { unreachable!("not called by `update_recursively` and must not be called by anything else") } @@ -210,7 +209,7 @@ impl DataSource for () { _cx: &UpdateContext<'_>, _range: UniversalRange>, _remote_fetch_timer: &mut AggregateTimer, - ) -> Result { + ) -> Result { Ok(()) } } @@ -234,14 +233,14 @@ macro_rules! impl_data_source_for_tuple { None } - async fn update_recursively(cx: &UpdateContext<'_>) -> Result<(), UpdateError> { + async fn update_recursively(cx: &UpdateContext<'_>) -> Result<(), ChartError> { $( $element_generic_name::update_recursively(cx).await?; )+ Ok(()) } - async fn update_itself(_cx: &UpdateContext<'_>) -> Result<(), UpdateError> { + async fn update_itself(_cx: &UpdateContext<'_>) -> Result<(), ChartError> { // dependencies are called in `update_recursively` // the tuple itself does not need any init Ok(()) @@ -251,7 +250,7 @@ macro_rules! impl_data_source_for_tuple { cx: &UpdateContext<'_>, range: UniversalRange>, remote_fetch_timer: &mut AggregateTimer, - ) -> Result { + ) -> Result { Ok(( $( $element_generic_name::query_data(cx, range.clone(), remote_fetch_timer).await? diff --git a/stats/stats/src/data_source/tests.rs b/stats/stats/src/data_source/tests.rs index 480d7cbf7..569537f0d 100644 --- a/stats/stats/src/data_source/tests.rs +++ b/stats/stats/src/data_source/tests.rs @@ -38,7 +38,7 @@ use crate::{ types::timespans::{DateValue, Month, Week, Year}, update_group::{SyncUpdateGroup, UpdateGroup}, utils::{produce_filter_and_values, sql_with_range_filter_opt}, - ChartProperties, MissingDatePolicy, Named, UpdateError, + ChartProperties, MissingDatePolicy, Named, ChartError, }; pub struct NewContractsQuery; @@ -217,7 +217,7 @@ impl BatchStepBehaviour>, ()> _last_accurate_point: DateValue, _main_data: Vec>, _resolution_data: (), - ) -> Result { + ) -> Result { // do something (just an example, not intended for running) todo!(); // save data diff --git a/stats/stats/src/lib.rs b/stats/stats/src/lib.rs index fa477611b..f53b02971 100644 --- a/stats/stats/src/lib.rs +++ b/stats/stats/src/lib.rs @@ -23,5 +23,5 @@ pub use charts::{ RequestedPointsLimit, }, lines, query_dispatch, types, ChartKey, ChartObject, ChartProperties, ChartPropertiesObject, - MissingDatePolicy, Named, ResolutionKind, UpdateError, + MissingDatePolicy, Named, ResolutionKind, ChartError, }; diff --git a/stats/stats/src/range.rs b/stats/stats/src/range.rs index 9efb733ed..612b013a3 100644 --- a/stats/stats/src/range.rs +++ b/stats/stats/src/range.rs @@ -5,7 +5,7 @@ use chrono::{DateTime, Utc}; use crate::{ data_source::{kinds::remote_db::RemoteQueryBehaviour, UpdateContext}, types::{Timespan, TimespanDuration}, - UpdateError, + ChartError, }; #[derive(Debug, Clone, PartialEq, Eq)] @@ -298,7 +298,7 @@ impl Decrementable for i32 { pub async fn data_source_query_range_to_db_statement_range( cx: &UpdateContext<'_>, data_source_range: UniversalRange>, -) -> Result>>, UpdateError> +) -> Result>>, ChartError> where AllRangeSource: RemoteQueryBehaviour>>, { diff --git a/stats/stats/src/update_group.rs b/stats/stats/src/update_group.rs index ec8793c19..f33d72ba3 100644 --- a/stats/stats/src/update_group.rs +++ b/stats/stats/src/update_group.rs @@ -43,7 +43,7 @@ use tokio::sync::{Mutex, MutexGuard}; use crate::{ charts::{chart_properties_portrait::imports::ChartKey, ChartObject}, data_source::UpdateParameters, - UpdateError, + ChartError, }; #[derive(Error, Debug, PartialEq)] @@ -95,7 +95,7 @@ pub trait UpdateGroup: core::fmt::Debug { &self, params: UpdateParameters<'a>, enabled_charts: &HashSet, - ) -> Result<(), UpdateError>; + ) -> Result<(), ChartError>; } /// Construct update group that implemants [`UpdateGroup`]. The main purpose of the @@ -117,7 +117,7 @@ pub trait UpdateGroup: core::fmt::Debug { /// ```rust /// # use stats::data_source::kinds::{ /// # }; -/// # use stats::{ChartProperties, Named, construct_update_group, types::timespans::DateValue, UpdateError}; +/// # use stats::{ChartProperties, Named, construct_update_group, types::timespans::DateValue, ChartError}; /// # use stats::data_source::{ /// # kinds::{ /// # local_db::{DirectVecLocalDbChartSource, parameters::update::batching::parameters::Batch30Days}, @@ -198,7 +198,7 @@ pub trait UpdateGroup: core::fmt::Debug { /// ```rust /// # use stats::{ /// # QueryAllBlockTimestampRange, construct_update_group, -/// # types::timespans::DateValue, ChartProperties, Named, UpdateError, +/// # types::timespans::DateValue, ChartProperties, Named, ChartError, /// # }; /// # use stats::data_source::{ /// # kinds::{ @@ -330,7 +330,7 @@ macro_rules! construct_update_group { params: $crate::data_source::UpdateParameters<'a>, #[allow(unused)] enabled_charts: &::std::collections::HashSet<$crate::ChartKey>, - ) -> Result<(), $crate::UpdateError> { + ) -> Result<(), $crate::ChartError> { let cx = $crate::data_source::UpdateContext::from_params_now_or_override(params); ::tracing::Span::current().record("update_time", ::std::format!("{}",&cx.time)); $( @@ -500,12 +500,12 @@ impl SyncUpdateGroup { db: &DatabaseConnection, creation_time_override: Option>, enabled_charts: &HashSet, - ) -> Result<(), UpdateError> { + ) -> Result<(), ChartError> { let (_joint_guard, enabled_members) = self.lock_enabled_dependencies(enabled_charts).await; self.inner .create_charts(db, creation_time_override, &enabled_members) .await - .map_err(UpdateError::StatsDB) + .map_err(ChartError::StatsDB) } /// Ignores unknown names @@ -513,7 +513,7 @@ impl SyncUpdateGroup { &self, params: UpdateParameters<'a>, enabled_charts: &HashSet, - ) -> Result<(), UpdateError> { + ) -> Result<(), ChartError> { let (_joint_guard, enabled_members) = self.lock_enabled_dependencies(enabled_charts).await; tracing::info!( update_group = self.name(), diff --git a/stats/stats/src/utils.rs b/stats/stats/src/utils.rs index 8220dffda..1d1f937d5 100644 --- a/stats/stats/src/utils.rs +++ b/stats/stats/src/utils.rs @@ -12,6 +12,9 @@ pub fn day_start(date: &NaiveDate) -> DateTime { .and_utc() } +/// Database connection with a mark of what database it is. +/// Used to separate caching for different databases to +/// prevent data clashes when running unit tests concurrently. #[derive(Debug, Clone)] pub struct MarkedDbConnection { pub connection: Arc, From 14ab9b430994dfa277dfeaf6401665136c09ec9a Mon Sep 17 00:00:00 2001 From: Kirill Ivanov Date: Wed, 4 Dec 2024 19:58:35 +0900 Subject: [PATCH 27/41] fix incorrect directive used --- stats/stats/src/utils.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stats/stats/src/utils.rs b/stats/stats/src/utils.rs index 1d1f937d5..d236ea297 100644 --- a/stats/stats/src/utils.rs +++ b/stats/stats/src/utils.rs @@ -22,7 +22,7 @@ pub struct MarkedDbConnection { } impl MarkedDbConnection { - #![cfg(any(feature = "test-utils", test))] + #[cfg(any(feature = "test-utils", test))] pub fn from_test_db( guard: &blockscout_service_launcher::test_database::TestDbGuard, ) -> Option { From 791c382874442ccb6286acc0e9c9db43db26bbee Mon Sep 17 00:00:00 2001 From: Kirill Ivanov Date: Wed, 4 Dec 2024 20:17:51 +0900 Subject: [PATCH 28/41] fmt --- stats/stats-server/src/read_service.rs | 2 +- stats/stats/src/charts/counters/average_block_time.rs | 2 +- stats/stats/src/charts/counters/mock.rs | 2 +- stats/stats/src/charts/counters/total_blocks.rs | 2 +- stats/stats/src/charts/counters/total_contracts.rs | 2 +- stats/stats/src/charts/counters/yesterday_txns.rs | 2 +- stats/stats/src/charts/db_interaction/read.rs | 2 +- stats/stats/src/charts/lines/mock.rs | 2 +- stats/stats/src/charts/lines/native_coin_holders_growth.rs | 2 +- stats/stats/src/charts/mod.rs | 4 ++-- stats/stats/src/charts/query_dispatch.rs | 2 +- .../data_source/kinds/data_manipulation/filter_deducible.rs | 2 +- .../src/data_source/kinds/data_manipulation/map/parse.rs | 4 +--- stats/stats/src/data_source/kinds/local_db/mod.rs | 2 +- .../stats/src/data_source/kinds/local_db/parameter_traits.rs | 2 +- .../stats/src/data_source/kinds/local_db/parameters/query.rs | 4 ++-- .../kinds/local_db/parameters/update/batching/mod.rs | 2 +- .../parameters/update/batching/parameters/cumulative.rs | 2 +- stats/stats/src/data_source/tests.rs | 2 +- stats/stats/src/lib.rs | 4 ++-- 20 files changed, 23 insertions(+), 25 deletions(-) diff --git a/stats/stats-server/src/read_service.rs b/stats/stats-server/src/read_service.rs index 5784db94d..9ac964344 100644 --- a/stats/stats-server/src/read_service.rs +++ b/stats/stats-server/src/read_service.rs @@ -20,7 +20,7 @@ use stats::{ range::UniversalRange, types::Timespan, utils::{day_start, MarkedDbConnection}, - RequestedPointsLimit, ResolutionKind, ChartError, + ChartError, RequestedPointsLimit, ResolutionKind, }; use stats_proto::blockscout::stats::v1 as proto_v1; use tonic::{Request, Response, Status}; diff --git a/stats/stats/src/charts/counters/average_block_time.rs b/stats/stats/src/charts/counters/average_block_time.rs index 67c76a1f9..3d865cc36 100644 --- a/stats/stats/src/charts/counters/average_block_time.rs +++ b/stats/stats/src/charts/counters/average_block_time.rs @@ -12,7 +12,7 @@ use crate::{ range::UniversalRange, types::TimespanValue, utils::NANOS_PER_SEC, - ChartProperties, MissingDatePolicy, Named, ChartError, + ChartError, ChartProperties, MissingDatePolicy, Named, }; use blockscout_db::entity::blocks; diff --git a/stats/stats/src/charts/counters/mock.rs b/stats/stats/src/charts/counters/mock.rs index 701e2aace..d29b51d1c 100644 --- a/stats/stats/src/charts/counters/mock.rs +++ b/stats/stats/src/charts/counters/mock.rs @@ -11,7 +11,7 @@ use crate::{ }, range::UniversalRange, types::timespans::DateValue, - ChartProperties, Named, ChartError, + ChartError, ChartProperties, Named, }; use chrono::{DateTime, NaiveDate, Utc}; diff --git a/stats/stats/src/charts/counters/total_blocks.rs b/stats/stats/src/charts/counters/total_blocks.rs index 24cdc94a1..41f8cde8b 100644 --- a/stats/stats/src/charts/counters/total_blocks.rs +++ b/stats/stats/src/charts/counters/total_blocks.rs @@ -12,7 +12,7 @@ use crate::{ range::UniversalRange, types::timespans::DateValue, utils::MarkedDbConnection, - ChartProperties, MissingDatePolicy, Named, ChartError, + ChartError, ChartProperties, MissingDatePolicy, Named, }; use blockscout_db::entity::blocks; diff --git a/stats/stats/src/charts/counters/total_contracts.rs b/stats/stats/src/charts/counters/total_contracts.rs index c5eeeab67..841d7b6a1 100644 --- a/stats/stats/src/charts/counters/total_contracts.rs +++ b/stats/stats/src/charts/counters/total_contracts.rs @@ -8,7 +8,7 @@ use crate::{ }, range::UniversalRange, types::timespans::DateValue, - ChartProperties, MissingDatePolicy, Named, ChartError, + ChartError, ChartProperties, MissingDatePolicy, Named, }; use blockscout_db::entity::addresses; diff --git a/stats/stats/src/charts/counters/yesterday_txns.rs b/stats/stats/src/charts/counters/yesterday_txns.rs index f16f39cc8..691ee7b97 100644 --- a/stats/stats/src/charts/counters/yesterday_txns.rs +++ b/stats/stats/src/charts/counters/yesterday_txns.rs @@ -10,7 +10,7 @@ use crate::{ range::UniversalRange, types::TimespanValue, utils::day_start, - ChartProperties, MissingDatePolicy, Named, ChartError, + ChartError, ChartProperties, MissingDatePolicy, Named, }; use chrono::{DateTime, Days, NaiveDate, Utc}; use entity::sea_orm_active_enums::ChartType; diff --git a/stats/stats/src/charts/db_interaction/read.rs b/stats/stats/src/charts/db_interaction/read.rs index 7b55f639a..4c02cf0cd 100644 --- a/stats/stats/src/charts/db_interaction/read.rs +++ b/stats/stats/src/charts/db_interaction/read.rs @@ -10,7 +10,7 @@ use crate::{ timespans::{DateValue, Month, Week, Year}, ExtendedTimespanValue, Timespan, TimespanDuration, TimespanValue, }, - ChartProperties, MissingDatePolicy, ChartError, + ChartError, ChartProperties, MissingDatePolicy, }; use blockscout_db::entity::blocks; diff --git a/stats/stats/src/charts/lines/mock.rs b/stats/stats/src/charts/lines/mock.rs index ac6b5a151..af59c0878 100644 --- a/stats/stats/src/charts/lines/mock.rs +++ b/stats/stats/src/charts/lines/mock.rs @@ -12,7 +12,7 @@ use crate::{ missing_date::fit_into_range, range::{Incrementable, UniversalRange}, types::{timespans::DateValue, Timespan, TimespanValue}, - ChartProperties, MissingDatePolicy, Named, ChartError, + ChartError, ChartProperties, MissingDatePolicy, Named, }; use chrono::{DateTime, Duration, NaiveDate, Utc}; diff --git a/stats/stats/src/charts/lines/native_coin_holders_growth.rs b/stats/stats/src/charts/lines/native_coin_holders_growth.rs index f9b82e997..9e2fc8c4e 100644 --- a/stats/stats/src/charts/lines/native_coin_holders_growth.rs +++ b/stats/stats/src/charts/lines/native_coin_holders_growth.rs @@ -21,7 +21,7 @@ use crate::{ }, define_and_impl_resolution_properties, types::timespans::{DateValue, Month, Week, Year}, - ChartProperties, MissingDatePolicy, Named, ChartError, + ChartError, ChartProperties, MissingDatePolicy, Named, }; use blockscout_db::entity::address_coin_balances_daily; diff --git a/stats/stats/src/charts/mod.rs b/stats/stats/src/charts/mod.rs index 510e7b7e9..75e9b3c23 100644 --- a/stats/stats/src/charts/mod.rs +++ b/stats/stats/src/charts/mod.rs @@ -5,6 +5,6 @@ pub mod lines; pub mod query_dispatch; pub mod types; pub use chart::{ - chart_properties_portrait, ChartKey, ChartObject, ChartProperties, ChartPropertiesObject, - MissingDatePolicy, Named, ResolutionKind, ChartError, + chart_properties_portrait, ChartError, ChartKey, ChartObject, ChartProperties, + ChartPropertiesObject, MissingDatePolicy, Named, ResolutionKind, }; diff --git a/stats/stats/src/charts/query_dispatch.rs b/stats/stats/src/charts/query_dispatch.rs index 997bd7b12..fe67f4dbd 100644 --- a/stats/stats/src/charts/query_dispatch.rs +++ b/stats/stats/src/charts/query_dispatch.rs @@ -18,7 +18,7 @@ use crate::{ use super::{ types::{ExtendedTimespanValue, Timespan, TimespanValue}, - ChartProperties, ChartError, + ChartError, ChartProperties, }; /// Data query trait with unified data format (for external use) diff --git a/stats/stats/src/data_source/kinds/data_manipulation/filter_deducible.rs b/stats/stats/src/data_source/kinds/data_manipulation/filter_deducible.rs index 28728ffa4..a7b4581b7 100644 --- a/stats/stats/src/data_source/kinds/data_manipulation/filter_deducible.rs +++ b/stats/stats/src/data_source/kinds/data_manipulation/filter_deducible.rs @@ -12,7 +12,7 @@ use crate::{ data_source::{DataSource, UpdateContext}, range::UniversalRange, types::TimespanValue, - ChartProperties, MissingDatePolicy, ChartError, + ChartError, ChartProperties, MissingDatePolicy, }; /// Pass only essential points from `D`, removing ones that can be deduced diff --git a/stats/stats/src/data_source/kinds/data_manipulation/map/parse.rs b/stats/stats/src/data_source/kinds/data_manipulation/map/parse.rs index c1a9b88ca..dfe3f5fd4 100644 --- a/stats/stats/src/data_source/kinds/data_manipulation/map/parse.rs +++ b/stats/stats/src/data_source/kinds/data_manipulation/map/parse.rs @@ -43,9 +43,7 @@ where { type Output = TimespanValue; - fn function( - inner_data: TimespanValue, - ) -> Result { + fn function(inner_data: TimespanValue) -> Result { let val_parsed = inner_data.value.parse::().map_err(|e| { ChartError::Internal(format!("failed to parse values of dependency: {e}")) })?; diff --git a/stats/stats/src/data_source/kinds/local_db/mod.rs b/stats/stats/src/data_source/kinds/local_db/mod.rs index 2cabd4282..93cc00694 100644 --- a/stats/stats/src/data_source/kinds/local_db/mod.rs +++ b/stats/stats/src/data_source/kinds/local_db/mod.rs @@ -329,7 +329,7 @@ mod tests { tests::{init_db::init_marked_db_all, mock_blockscout::fill_mock_blockscout_data}, types::{timespans::DateValue, TimespanValue}, update_group::{SyncUpdateGroup, UpdateGroup}, - ChartProperties, Named, ChartError, + ChartError, ChartProperties, Named, }; type WasTriggeredStorage = Arc>; diff --git a/stats/stats/src/data_source/kinds/local_db/parameter_traits.rs b/stats/stats/src/data_source/kinds/local_db/parameter_traits.rs index 69a62a545..d62888188 100644 --- a/stats/stats/src/data_source/kinds/local_db/parameter_traits.rs +++ b/stats/stats/src/data_source/kinds/local_db/parameter_traits.rs @@ -9,7 +9,7 @@ use crate::{ data_source::{DataSource, UpdateContext}, range::UniversalRange, types::TimespanValue, - RequestedPointsLimit, ChartError, + ChartError, RequestedPointsLimit, }; /// In most cases, [`super::DefaultCreate`] is enough. diff --git a/stats/stats/src/data_source/kinds/local_db/parameters/query.rs b/stats/stats/src/data_source/kinds/local_db/parameters/query.rs index 85268f1f7..c47738fb6 100644 --- a/stats/stats/src/data_source/kinds/local_db/parameters/query.rs +++ b/stats/stats/src/data_source/kinds/local_db/parameters/query.rs @@ -9,7 +9,7 @@ use crate::{ range::UniversalRange, types::{timespans::DateValue, ExtendedTimespanValue, Timespan}, utils::MarkedDbConnection, - ChartProperties, RequestedPointsLimit, ChartError, + ChartError, ChartProperties, RequestedPointsLimit, }; /// Usually the choice for line charts @@ -142,7 +142,7 @@ mod tests { data_source::{types::BlockscoutMigrations, UpdateContext, UpdateParameters}, tests::init_db::init_marked_db_all, types::timespans::DateValue, - MissingDatePolicy, Named, ChartError, + ChartError, MissingDatePolicy, Named, }; #[tokio::test] diff --git a/stats/stats/src/data_source/kinds/local_db/parameters/update/batching/mod.rs b/stats/stats/src/data_source/kinds/local_db/parameters/update/batching/mod.rs index 9b1d9ab0d..d4e83821d 100644 --- a/stats/stats/src/data_source/kinds/local_db/parameters/update/batching/mod.rs +++ b/stats/stats/src/data_source/kinds/local_db/parameters/update/batching/mod.rs @@ -19,7 +19,7 @@ use crate::{ }, range::UniversalRange, types::{ExtendedTimespanValue, Timespan, TimespanDuration, TimespanValue}, - ChartProperties, ChartError, + ChartError, ChartProperties, }; pub mod parameter_traits; diff --git a/stats/stats/src/data_source/kinds/local_db/parameters/update/batching/parameters/cumulative.rs b/stats/stats/src/data_source/kinds/local_db/parameters/update/batching/parameters/cumulative.rs index c01894e68..ffaad7478 100644 --- a/stats/stats/src/data_source/kinds/local_db/parameters/update/batching/parameters/cumulative.rs +++ b/stats/stats/src/data_source/kinds/local_db/parameters/update/batching/parameters/cumulative.rs @@ -9,7 +9,7 @@ use sea_orm::DatabaseConnection; use crate::{ data_source::kinds::local_db::parameters::update::batching::parameter_traits::BatchStepBehaviour, types::{Timespan, TimespanValue}, - ChartProperties, ChartError, + ChartError, ChartProperties, }; use super::PassVecStep; diff --git a/stats/stats/src/data_source/tests.rs b/stats/stats/src/data_source/tests.rs index 569537f0d..f48419e20 100644 --- a/stats/stats/src/data_source/tests.rs +++ b/stats/stats/src/data_source/tests.rs @@ -38,7 +38,7 @@ use crate::{ types::timespans::{DateValue, Month, Week, Year}, update_group::{SyncUpdateGroup, UpdateGroup}, utils::{produce_filter_and_values, sql_with_range_filter_opt}, - ChartProperties, MissingDatePolicy, Named, ChartError, + ChartError, ChartProperties, MissingDatePolicy, Named, }; pub struct NewContractsQuery; diff --git a/stats/stats/src/lib.rs b/stats/stats/src/lib.rs index f53b02971..dfe057c0b 100644 --- a/stats/stats/src/lib.rs +++ b/stats/stats/src/lib.rs @@ -22,6 +22,6 @@ pub use charts::{ get_line_chart_data, ApproxUnsignedDiff, QueryAllBlockTimestampRange, ReadError, RequestedPointsLimit, }, - lines, query_dispatch, types, ChartKey, ChartObject, ChartProperties, ChartPropertiesObject, - MissingDatePolicy, Named, ResolutionKind, ChartError, + lines, query_dispatch, types, ChartError, ChartKey, ChartObject, ChartProperties, + ChartPropertiesObject, MissingDatePolicy, Named, ResolutionKind, }; From fae1d75d0151218ef55e059a231fc18bf72b652d Mon Sep 17 00:00:00 2001 From: Kirill Ivanov Date: Wed, 4 Dec 2024 23:47:36 +0900 Subject: [PATCH 29/41] don't use raw db read for chart tests --- stats/stats/src/tests/simple_test.rs | 131 ++++++++------------------- 1 file changed, 40 insertions(+), 91 deletions(-) diff --git a/stats/stats/src/tests/simple_test.rs b/stats/stats/src/tests/simple_test.rs index 482b80855..0f7af1ece 100644 --- a/stats/stats/src/tests/simple_test.rs +++ b/stats/stats/src/tests/simple_test.rs @@ -7,17 +7,16 @@ use crate::{ source::DataSource, types::{BlockscoutMigrations, UpdateContext, UpdateParameters}, }, - get_line_chart_data, query_dispatch::QuerySerialized, range::UniversalRange, types::{timespans::DateValue, Timespan}, utils::MarkedDbConnection, - ChartProperties, MissingDatePolicy, + ChartProperties, }; use blockscout_service_launcher::test_database::TestDbGuard; use chrono::{DateTime, NaiveDateTime, Utc}; use pretty_assertions::assert_eq; -use sea_orm::DatabaseConnection; +use stats_proto::blockscout::stats::v1::Point; use std::{fmt::Debug, str::FromStr}; pub fn map_str_tuple_to_owned(l: Vec<(&str, &str)>) -> Vec<(String, String)> { @@ -39,7 +38,7 @@ pub async fn simple_test_chart( expected: Vec<(&str, &str)>, ) -> (TestDbGuard, TestDbGuard) where - C: DataSource + ChartProperties, + C: DataSource + ChartProperties + QuerySerialized>, C::Resolution: Ord + Clone + Debug, { simple_test_chart_inner::(test_name, expected, BlockscoutMigrations::latest()).await @@ -56,7 +55,7 @@ pub async fn simple_test_chart_with_migration_variants( test_name_base: &str, expected: Vec<(&str, &str)>, ) where - C: DataSource + ChartProperties, + C: DataSource + ChartProperties + QuerySerialized>, C::Resolution: Ord + Clone + Debug, { for (i, migrations) in MIGRATIONS_VARIANTS.into_iter().enumerate() { @@ -65,18 +64,21 @@ pub async fn simple_test_chart_with_migration_variants( } } +fn chart_output_to_expected(output: Vec) -> Vec<(String, String)> { + output.into_iter().map(|p| (p.date, p.value)).collect() +} + async fn simple_test_chart_inner( test_name: &str, expected: Vec<(&str, &str)>, migrations: BlockscoutMigrations, ) -> (TestDbGuard, TestDbGuard) where - C: DataSource + ChartProperties, + C: DataSource + ChartProperties + QuerySerialized>, C::Resolution: Ord + Clone + Debug, { let (current_time, db, blockscout) = prepare_chart_test::(test_name, None).await; let expected = map_str_tuple_to_owned(expected); - let approximate_trailing_points = C::approximate_trailing_points(); let current_date = current_time.date_naive(); fill_mock_blockscout_data(&blockscout, current_date).await; @@ -90,15 +92,11 @@ where let cx = UpdateContext::from_params_now_or_override(parameters.clone()); C::update_recursively(&cx).await.unwrap(); assert_eq!( - &get_chart::( - &db, - None, - None, - C::missing_date_policy(), - false, - approximate_trailing_points, - ) - .await, + &chart_output_to_expected( + C::query_data_static(&cx, UniversalRange::full(), None, false) + .await + .unwrap() + ), &expected ); @@ -106,15 +104,11 @@ where let cx = UpdateContext::from_params_now_or_override(parameters); C::update_recursively(&cx).await.unwrap(); assert_eq!( - &get_chart::( - &db, - None, - None, - C::missing_date_policy(), - false, - approximate_trailing_points, - ) - .await, + &chart_output_to_expected( + C::query_data_static(&cx, UniversalRange::full(), None, false) + .await + .unwrap() + ), &expected ); (db, blockscout) @@ -129,7 +123,7 @@ pub async fn dirty_force_update_and_check( expected: Vec<(&str, &str)>, update_time_override: Option>, ) where - C: DataSource + ChartProperties, + C: DataSource + ChartProperties + QuerySerialized>, C::Resolution: Ord + Clone + Debug, { let _ = tracing_subscriber::fmt::try_init(); @@ -137,7 +131,6 @@ pub async fn dirty_force_update_and_check( // some later time so that the update is not skipped let current_time = update_time_override.unwrap_or(DateTime::from_str("2023-03-01T12:00:01Z").unwrap()); - let approximate_trailing_points = C::approximate_trailing_points(); let parameters = UpdateParameters { db: &MarkedDbConnection::from_test_db(db).unwrap(), @@ -149,15 +142,11 @@ pub async fn dirty_force_update_and_check( let cx = UpdateContext::from_params_now_or_override(parameters.clone()); C::update_recursively(&cx).await.unwrap(); assert_eq!( - &get_chart::( - db, - None, - None, - C::missing_date_policy(), - false, - approximate_trailing_points, - ) - .await, + &chart_output_to_expected( + C::query_data_static(&cx, UniversalRange::full(), None, false) + .await + .unwrap() + ), &expected ); } @@ -176,7 +165,7 @@ pub async fn ranged_test_chart( to: C::Resolution, update_time: Option, ) where - C: DataSource + ChartProperties, + C: DataSource + ChartProperties + QuerySerialized>, C::Resolution: Ord + Clone + Debug, { ranged_test_chart_inner::( @@ -204,7 +193,7 @@ pub async fn ranged_test_chart_with_migration_variants( to: C::Resolution, update_time: Option, ) where - C: DataSource + ChartProperties, + C: DataSource + ChartProperties + QuerySerialized>, C::Resolution: Ord + Clone + Debug, { for (i, migrations) in MIGRATIONS_VARIANTS.into_iter().enumerate() { @@ -229,7 +218,7 @@ async fn ranged_test_chart_inner( update_time: Option, migrations: BlockscoutMigrations, ) where - C: DataSource + ChartProperties, + C: DataSource + ChartProperties + QuerySerialized>, C::Resolution: Ord + Clone + Debug, { let _ = tracing_subscriber::fmt::try_init(); @@ -238,12 +227,11 @@ async fn ranged_test_chart_inner( let max_time = DateTime::::from_str("2023-03-01T12:00:00Z").unwrap(); let current_time = update_time.map(|t| t.and_utc()).unwrap_or(max_time); let max_date = max_time.date_naive(); + let range = { from.into_time_range().start..to.into_time_range().end }; C::init_recursively(db.connection.as_ref(), ¤t_time) .await .unwrap(); fill_mock_blockscout_data(blockscout.connection.as_ref(), max_date).await; - let policy = C::missing_date_policy(); - let approximate_trailing_points = C::approximate_trailing_points(); let mut parameters = UpdateParameters { db: &db, @@ -255,15 +243,11 @@ async fn ranged_test_chart_inner( let cx = UpdateContext::from_params_now_or_override(parameters.clone()); C::update_recursively(&cx).await.unwrap(); assert_eq!( - &get_chart::( - db.connection.as_ref(), - Some(from.clone()), - Some(to.clone()), - policy, - false, - approximate_trailing_points, - ) - .await, + &chart_output_to_expected( + C::query_data_static(&cx, range.clone().into(), None, false) + .await + .unwrap() + ), &expected ); @@ -271,48 +255,15 @@ async fn ranged_test_chart_inner( let cx = UpdateContext::from_params_now_or_override(parameters); C::update_recursively(&cx).await.unwrap(); assert_eq!( - &get_chart::( - db.connection.as_ref(), - Some(from), - Some(to), - policy, - false, - approximate_trailing_points, - ) - .await, + &chart_output_to_expected( + C::query_data_static(&cx, range.into(), None, false) + .await + .unwrap() + ), &expected ); } -async fn get_chart( - db: &DatabaseConnection, - from: Option, - to: Option, - policy: MissingDatePolicy, - fill_missing_dates: bool, - approximate_trailing_points: u64, -) -> Vec<(String, String)> -where - C: DataSource + ChartProperties, - C::Resolution: Ord + Clone + Debug, -{ - let data = get_line_chart_data::( - db, - &C::name(), - from, - to, - None, - policy, - fill_missing_dates, - approximate_trailing_points, - ) - .await - .unwrap(); - data.into_iter() - .map(|p| (p.timespan.into_date().to_string(), p.value)) - .collect() -} - /// `test_name` must be unique to avoid db clashes pub async fn simple_test_counter( test_name: &str, @@ -395,9 +346,7 @@ pub async fn prepare_chart_test( pub async fn get_counter>>( cx: &UpdateContext<'_>, ) -> DateValue { - let counter_object = ::new_for_dynamic_dispatch(); - counter_object - .query_data(cx, UniversalRange::full(), None, false) + C::query_data_static(cx, UniversalRange::full(), None, false) .await .unwrap() } From 481a94c42f12b2370591c1dfe97ee0392843f6c4 Mon Sep 17 00:00:00 2001 From: Kirill Ivanov Date: Wed, 4 Dec 2024 23:52:35 +0900 Subject: [PATCH 30/41] add static version of query for convenience --- stats/stats/src/charts/query_dispatch.rs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/stats/stats/src/charts/query_dispatch.rs b/stats/stats/src/charts/query_dispatch.rs index fe67f4dbd..9cf11d312 100644 --- a/stats/stats/src/charts/query_dispatch.rs +++ b/stats/stats/src/charts/query_dispatch.rs @@ -40,6 +40,19 @@ pub trait QuerySerialized { points_limit: Option, fill_missing_dates: bool, ) -> Pin> + Send + 'a>>; + + /// Retrieve chart data from local storage. + fn query_data_static<'a>( + cx: &UpdateContext<'a>, + range: UniversalRange>, + points_limit: Option, + fill_missing_dates: bool, + ) -> Pin> + Send + 'a>> + where + Self: Sized, + { + Self::new_for_dynamic_dispatch().query_data(cx, range, points_limit, fill_missing_dates) + } } /// [`QuerySerialized`] but for dynamic dispatch From 700832057cfb2bb209dcc06e01b7744adc45a44e Mon Sep 17 00:00:00 2001 From: Kirill Ivanov Date: Thu, 5 Dec 2024 00:08:49 +0900 Subject: [PATCH 31/41] eradicate direct DB access where shouldn't --- stats/stats/src/charts/lines/new_blocks.rs | 73 +++++-------------- .../kinds/local_db/parameters/query.rs | 3 +- stats/stats/src/lib.rs | 3 +- 3 files changed, 21 insertions(+), 58 deletions(-) diff --git a/stats/stats/src/charts/lines/new_blocks.rs b/stats/stats/src/charts/lines/new_blocks.rs index 373745455..edcc33045 100644 --- a/stats/stats/src/charts/lines/new_blocks.rs +++ b/stats/stats/src/charts/lines/new_blocks.rs @@ -105,7 +105,8 @@ mod tests { use crate::{ charts::db_interaction::read::get_min_block_blockscout, data_source::{types::BlockscoutMigrations, DataSource, UpdateContext}, - get_line_chart_data, + query_dispatch::{serialize_line_points, QuerySerialized}, + range::UniversalRange, tests::{ init_db::init_marked_db_all, mock_blockscout::fill_mock_blockscout_data, point_construction::dt, simple_test::simple_test_chart, @@ -175,18 +176,9 @@ mod tests { force_full: false, }; NewBlocks::update_recursively(&cx).await.unwrap(); - let data = get_line_chart_data::( - db.connection.as_ref(), - &NewBlocks::name(), - None, - None, - None, - crate::MissingDatePolicy::FillZero, - false, - 1, - ) - .await - .unwrap(); + let data = NewBlocks::query_data_static(&cx, UniversalRange::full(), None, false) + .await + .unwrap(); let expected = vec![ ExtendedTimespanValue { timespan: NaiveDate::from_str("2022-11-10").unwrap(), @@ -204,25 +196,16 @@ mod tests { is_approximate: true, }, ]; - assert_eq!(expected, data); + assert_eq!(serialize_line_points(expected), data); // note that update is full, therefore there is entry with date `2022-11-09` cx.force_full = true; // need to update time so that the update is not ignored as the same one cx.time = chrono::DateTime::::from_str("2022-11-12T13:00:00Z").unwrap(); NewBlocks::update_recursively(&cx).await.unwrap(); - let data = get_line_chart_data::( - db.connection.as_ref(), - &NewBlocks::name(), - None, - None, - None, - crate::MissingDatePolicy::FillZero, - false, - 1, - ) - .await - .unwrap(); + let data = NewBlocks::query_data_static(&cx, UniversalRange::full(), None, false) + .await + .unwrap(); let expected = vec![ ExtendedTimespanValue { timespan: NaiveDate::from_str("2022-11-09").unwrap(), @@ -245,7 +228,7 @@ mod tests { is_approximate: true, }, ]; - assert_eq!(expected, data); + assert_eq!(serialize_line_points(expected), data); } #[tokio::test] @@ -269,18 +252,9 @@ mod tests { force_full: true, }; NewBlocks::update_recursively(&cx).await.unwrap(); - let data = get_line_chart_data::( - db.connection.as_ref(), - &NewBlocks::name(), - None, - None, - None, - crate::MissingDatePolicy::FillZero, - false, - 0, - ) - .await - .unwrap(); + let data = NewBlocks::query_data_static(&cx, UniversalRange::full(), None, false) + .await + .unwrap(); let expected = vec![ ExtendedTimespanValue { timespan: NaiveDate::from_str("2022-11-09").unwrap(), @@ -300,10 +274,10 @@ mod tests { ExtendedTimespanValue { timespan: NaiveDate::from_str("2022-11-12").unwrap(), value: "1".into(), - is_approximate: false, + is_approximate: true, }, ]; - assert_eq!(expected, data); + assert_eq!(serialize_line_points(expected), data); } #[tokio::test] @@ -375,18 +349,9 @@ mod tests { force_full: false, }; NewBlocks::update_recursively(&cx).await.unwrap(); - let data = get_line_chart_data::( - db.connection.as_ref(), - &NewBlocks::name(), - None, - None, - None, - crate::MissingDatePolicy::FillZero, - false, - 1, - ) - .await - .unwrap(); + let data = NewBlocks::query_data_static(&cx, UniversalRange::full(), None, false) + .await + .unwrap(); let expected = vec![ ExtendedTimespanValue { timespan: NaiveDate::from_str("2022-11-09").unwrap(), @@ -409,7 +374,7 @@ mod tests { is_approximate: true, }, ]; - assert_eq!(expected, data); + assert_eq!(serialize_line_points(expected), data); } #[tokio::test] diff --git a/stats/stats/src/data_source/kinds/local_db/parameters/query.rs b/stats/stats/src/data_source/kinds/local_db/parameters/query.rs index c47738fb6..b39ed7a4b 100644 --- a/stats/stats/src/data_source/kinds/local_db/parameters/query.rs +++ b/stats/stats/src/data_source/kinds/local_db/parameters/query.rs @@ -3,9 +3,8 @@ use std::{fmt::Debug, marker::PhantomData}; use chrono::{DateTime, Utc}; use crate::{ - charts::db_interaction::read::get_counter_data, + charts::db_interaction::read::{get_counter_data, get_line_chart_data}, data_source::{kinds::local_db::parameter_traits::QueryBehaviour, UpdateContext}, - get_line_chart_data, range::UniversalRange, types::{timespans::DateValue, ExtendedTimespanValue, Timespan}, utils::MarkedDbConnection, diff --git a/stats/stats/src/lib.rs b/stats/stats/src/lib.rs index dfe057c0b..27fb5a1ac 100644 --- a/stats/stats/src/lib.rs +++ b/stats/stats/src/lib.rs @@ -19,8 +19,7 @@ pub use migration; pub use charts::{ counters, db_interaction::read::{ - get_line_chart_data, ApproxUnsignedDiff, QueryAllBlockTimestampRange, ReadError, - RequestedPointsLimit, + ApproxUnsignedDiff, QueryAllBlockTimestampRange, ReadError, RequestedPointsLimit, }, lines, query_dispatch, types, ChartError, ChartKey, ChartObject, ChartProperties, ChartPropertiesObject, MissingDatePolicy, Named, ResolutionKind, From add91ce486c04996ecf2035871da64708bf73866 Mon Sep 17 00:00:00 2001 From: Kirill Ivanov Date: Thu, 5 Dec 2024 00:16:18 +0900 Subject: [PATCH 32/41] resolve todo 1 --- stats/stats/src/range.rs | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/stats/stats/src/range.rs b/stats/stats/src/range.rs index 612b013a3..107abc918 100644 --- a/stats/stats/src/range.rs +++ b/stats/stats/src/range.rs @@ -97,7 +97,6 @@ impl UniversalRange { /// None only if no backup is provided fn into_exclusive_inner(self, backup_bounds: Option>) -> Option> { let (backup_start, backup_end) = backup_bounds.map(|b| (b.start, b.end)).unzip(); - // todo: test that none backup and some start works as expected (+ end) let start = self.start.or(backup_start)?; match self.end { Bound::Included(end) => Some(inclusive_range_to_exclusive(start..=end)), @@ -371,6 +370,26 @@ mod tests { Some(d("2023-01-01")..d("2023-01-15")) ); + let partial: UniversalRange = UniversalRange { + start: None, + end: Bound::Excluded(d("2023-01-15")), + }; + assert_eq!(partial.clone().try_into_exclusive(), None); + assert_eq!( + partial + .clone() + .into_exclusive_with_backup(d("2023-01-01")..d("2023-01-16")), + d("2023-01-01")..d("2023-01-15") + ); + assert_eq!( + UniversalRange { + start: Some(d("2023-01-01")), + end: Bound::Unbounded, + } + .into_exclusive_with_backup(d("2023-01-02")..d("2023-01-16")), + d("2023-01-01")..d("2023-01-16") + ); + let unbounded = UniversalRange::full(); assert_eq!(unbounded.clone().try_into_exclusive(), None); From 5f88ce54445cffeba368098a85cf635ddd2d88c6 Mon Sep 17 00:00:00 2001 From: Kirill Ivanov Date: Thu, 5 Dec 2024 00:31:07 +0900 Subject: [PATCH 33/41] comments to cached impl --- stats/stats/src/charts/counters/total_blocks.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/stats/stats/src/charts/counters/total_blocks.rs b/stats/stats/src/charts/counters/total_blocks.rs index 41f8cde8b..bb8434dd0 100644 --- a/stats/stats/src/charts/counters/total_blocks.rs +++ b/stats/stats/src/charts/counters/total_blocks.rs @@ -86,6 +86,12 @@ static CACHED_BLOCKS_ESTIMATION: OnceLock> impl ValueEstimation for CachedBlocksEstimation { async fn estimate(blockscout: &MarkedDbConnection) -> Result, ChartError> { + /// Basically `cached::proc_macro::cached` implementation but + /// - manual + /// - using OnceLock value instead of hardcoded one + /// - expanding `result=true` to Result> + /// + /// Note, that `sync_writes = true` version was taken as a basis async fn cached_blocks_estimation( blockscout: &DatabaseConnection, db_id: &str, From cf3af02de347492a898269e205fe23cb3d743a22 Mon Sep 17 00:00:00 2001 From: Kirill Ivanov Date: Thu, 5 Dec 2024 00:31:12 +0900 Subject: [PATCH 34/41] fix test name --- stats/stats/src/charts/db_interaction/read.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stats/stats/src/charts/db_interaction/read.rs b/stats/stats/src/charts/db_interaction/read.rs index 4c02cf0cd..ac67a2191 100644 --- a/stats/stats/src/charts/db_interaction/read.rs +++ b/stats/stats/src/charts/db_interaction/read.rs @@ -826,10 +826,10 @@ mod tests { #[tokio::test] #[ignore = "needs database to run"] - async fn get_counters_mock() { + async fn get_counter_mock() { let _ = tracing_subscriber::fmt::try_init(); - let db = MarkedDbConnection::from_test_db(&init_db("get_counters_mock").await).unwrap(); + let db = MarkedDbConnection::from_test_db(&init_db("get_counter_mock").await).unwrap(); insert_mock_data(&db.connection).await; let current_time = dt("2022-11-12T08:08:08").and_utc(); let date = current_time.date_naive(); From e4a582615f9cfa7a09a303747aeff7dc1d500b43 Mon Sep 17 00:00:00 2001 From: Kirill Ivanov Date: Thu, 5 Dec 2024 00:42:23 +0900 Subject: [PATCH 35/41] add year ops test --- .../stats/src/charts/types/timespans/year.rs | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/stats/stats/src/charts/types/timespans/year.rs b/stats/stats/src/charts/types/timespans/year.rs index cca7565bf..68779f1f8 100644 --- a/stats/stats/src/charts/types/timespans/year.rs +++ b/stats/stats/src/charts/types/timespans/year.rs @@ -292,6 +292,53 @@ mod tests { Year::from_date(NaiveDate::MIN) ); } + #[test] + fn year_checked_add_works() { + assert_eq!( + Year(2016).checked_add(TimespanDuration::from_timespan_repeats(16)), + Some(Year(2032)) + ); + assert_eq!( + Year(2016).checked_add(TimespanDuration::from_timespan_repeats(0)), + Some(Year(2016)) + ); + assert_eq!( + Year(2016).checked_add(TimespanDuration::from_timespan_repeats(u64::MAX)), + None + ); + assert_eq!( + Year::from_date(NaiveDate::MAX).checked_add(TimespanDuration::from_timespan_repeats(1)), + None + ); + assert_eq!( + Year(i32::MAX - 5).checked_add(TimespanDuration::from_timespan_repeats(3)), + None + ); + } + + #[test] + fn year_checked_sub_works() { + assert_eq!( + Year(2016).checked_sub(TimespanDuration::from_timespan_repeats(16)), + Some(Year(2000)) + ); + assert_eq!( + Year(2016).checked_sub(TimespanDuration::from_timespan_repeats(0)), + Some(Year(2016)) + ); + assert_eq!( + Year(2016).checked_sub(TimespanDuration::from_timespan_repeats(u64::MAX)), + None + ); + assert_eq!( + Year::from_date(NaiveDate::MIN).checked_sub(TimespanDuration::from_timespan_repeats(1)), + None + ); + assert_eq!( + Year(i32::MIN + 5).checked_sub(TimespanDuration::from_timespan_repeats(3)), + None + ); + } #[test] fn year_saturating_first_timestamp_works() { From d954d8ea7fe914fe39ce9840c0d18386c70c9c2a Mon Sep 17 00:00:00 2001 From: Kirill Ivanov Date: Thu, 5 Dec 2024 00:55:58 +0900 Subject: [PATCH 36/41] remove unnecessary fn --- stats/stats/src/range.rs | 6 ------ 1 file changed, 6 deletions(-) diff --git a/stats/stats/src/range.rs b/stats/stats/src/range.rs index 107abc918..56b62790b 100644 --- a/stats/stats/src/range.rs +++ b/stats/stats/src/range.rs @@ -87,12 +87,6 @@ impl From>> for UniversalRange { } } -impl> UniversalRange { - pub fn is_unbounded(&self) -> bool { - self.start.is_none() || self.end == Bound::Unbounded - } -} - impl UniversalRange { /// None only if no backup is provided fn into_exclusive_inner(self, backup_bounds: Option>) -> Option> { From 76cee07637e5dcc1011b1fe1625cb1ebc4c8a3c6 Mon Sep 17 00:00:00 2001 From: Kirill Ivanov Date: Thu, 5 Dec 2024 00:58:47 +0900 Subject: [PATCH 37/41] update total blocks schedule to match blockscout's --- stats/config/update_groups.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stats/config/update_groups.json b/stats/config/update_groups.json index 870e8ba57..c07dab967 100644 --- a/stats/config/update_groups.json +++ b/stats/config/update_groups.json @@ -4,7 +4,7 @@ "average_block_time_group": "0 */10 * * * * *", "completed_txns_group": "0 5 */3 * * * *", "total_addresses_group": "0 0 */3 * * * *", - "total_blocks_group": "0 0 */3 * * * *", + "total_blocks_group": "0 0 */2 * * * *", "total_tokens_group": "0 0 18 * * * *", "yesterday_txns_group": "0 8 0 * * * *", "active_recurring_accounts_daily_recurrence_60_days_group": "0 0 2 * * * *", From b3595acf1af9d796dae248c8c7ad752308e28ad9 Mon Sep 17 00:00:00 2001 From: Kirill Ivanov Date: Mon, 9 Dec 2024 16:33:22 +0900 Subject: [PATCH 38/41] remove caching --- stats/Cargo.lock | 49 +------------ stats/Cargo.toml | 1 - stats/stats/Cargo.toml | 3 - .../stats/src/charts/counters/total_blocks.rs | 69 ++++--------------- stats/stats/src/utils.rs | 1 + 5 files changed, 15 insertions(+), 108 deletions(-) diff --git a/stats/Cargo.lock b/stats/Cargo.lock index ceb09f7f3..0b63993cf 100644 --- a/stats/Cargo.lock +++ b/stats/Cargo.lock @@ -1,6 +1,6 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 +version = 4 [[package]] name = "actix-codec" @@ -990,42 +990,6 @@ dependencies = [ "bytes", ] -[[package]] -name = "cached" -version = "0.54.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9718806c4a2fe9e8a56fd736f97b340dd10ed1be8ed733ed50449f351dc33cae" -dependencies = [ - "ahash 0.8.11", - "async-trait", - "cached_proc_macro", - "cached_proc_macro_types", - "futures", - "hashbrown 0.14.5", - "once_cell", - "thiserror", - "tokio", - "web-time", -] - -[[package]] -name = "cached_proc_macro" -version = "0.23.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f42a145ed2d10dce2191e1dcf30cfccfea9026660e143662ba5eec4017d5daa" -dependencies = [ - "darling", - "proc-macro2", - "quote", - "syn 2.0.79", -] - -[[package]] -name = "cached_proc_macro_types" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ade8366b8bd5ba243f0a58f036cc0ca8a2f069cff1a2351ef1cac6b083e16fc0" - [[package]] name = "cc" version = "1.0.98" @@ -4753,7 +4717,6 @@ dependencies = [ "blockscout-db", "blockscout-metrics-tools", "blockscout-service-launcher", - "cached", "chrono", "entity", "futures", @@ -5661,16 +5624,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "web-time" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" -dependencies = [ - "js-sys", - "wasm-bindgen", -] - [[package]] name = "webpki-roots" version = "0.25.4" diff --git a/stats/Cargo.toml b/stats/Cargo.toml index d019af282..82cf74972 100644 --- a/stats/Cargo.toml +++ b/stats/Cargo.toml @@ -12,7 +12,6 @@ members = [ [workspace.dependencies] blockscout-client = { git = "https://github.com/blockscout/blockscout-rs/", rev = "506b821" } blockscout-service-launcher = { version = "0.13.1" } -cached = "0.54" rstest = "0.23.0" trait-variant = "0.1.2" wiremock = "0.6.2" diff --git a/stats/stats/Cargo.toml b/stats/stats/Cargo.toml index dce42f435..ead654035 100644 --- a/stats/stats/Cargo.toml +++ b/stats/stats/Cargo.toml @@ -14,9 +14,6 @@ sea-orm = { version = "0.12", features = [ tokio = "1" thiserror = "1.0" chrono = "0.4" -cached = { workspace = true, features = [ - "async" -] } trait-variant = { workspace = true } paste = "1.0" portrait = "0.3.0" diff --git a/stats/stats/src/charts/counters/total_blocks.rs b/stats/stats/src/charts/counters/total_blocks.rs index bb8434dd0..4bca4811b 100644 --- a/stats/stats/src/charts/counters/total_blocks.rs +++ b/stats/stats/src/charts/counters/total_blocks.rs @@ -1,5 +1,3 @@ -use std::sync::OnceLock; - use crate::{ charts::db_interaction::read::query_estimated_table_rows, data_source::{ @@ -16,11 +14,9 @@ use crate::{ }; use blockscout_db::entity::blocks; -use cached::Cached; use chrono::{DateTime, NaiveDate, NaiveDateTime, Utc}; use entity::sea_orm_active_enums::ChartType; use sea_orm::{prelude::*, sea_query::Expr, FromQueryResult, QuerySelect}; -use tokio::sync::Mutex; #[derive(FromQueryResult)] struct TotalBlocksData { @@ -77,56 +73,20 @@ impl ChartProperties for Properties { } } -pub struct CachedBlocksEstimation; - -const TOTAL_BLOCKS_ESTIMATION_CACHE_LIVENESS_SEC_DEFAULT: u64 = 10; -// so that it can be added to settings if necessary -pub static TOTAL_BLOCKS_ESTIMATION_CACHE_LIVENESS_SEC: OnceLock = OnceLock::new(); -static CACHED_BLOCKS_ESTIMATION: OnceLock>> = OnceLock::new(); +pub struct TotalBlocksEstimation; -impl ValueEstimation for CachedBlocksEstimation { +impl ValueEstimation for TotalBlocksEstimation { async fn estimate(blockscout: &MarkedDbConnection) -> Result, ChartError> { - /// Basically `cached::proc_macro::cached` implementation but - /// - manual - /// - using OnceLock value instead of hardcoded one - /// - expanding `result=true` to Result> - /// - /// Note, that `sync_writes = true` version was taken as a basis - async fn cached_blocks_estimation( - blockscout: &DatabaseConnection, - db_id: &str, - ) -> Result, DbErr> { - let mut cache = CACHED_BLOCKS_ESTIMATION - .get_or_init(|| { - Mutex::new(cached::TimedCache::with_lifespan_and_refresh( - *TOTAL_BLOCKS_ESTIMATION_CACHE_LIVENESS_SEC - .get_or_init(|| TOTAL_BLOCKS_ESTIMATION_CACHE_LIVENESS_SEC_DEFAULT), - false, - )) - }) - .lock() - .await; - if let Some(cached_estimate) = cache.cache_get(db_id) { - return Ok(Some(*cached_estimate)); - } - - let query_result = - query_estimated_table_rows(blockscout, blocks::Entity.table_name()).await; - if let Ok(Some(estimate)) = &query_result { - cache.cache_set(db_id.to_string(), *estimate); - } - query_result - } - let now = Utc::now(); - let value = cached_blocks_estimation(blockscout.connection.as_ref(), &blockscout.db_name) - .await - .map_err(ChartError::BlockscoutDB)? - .map(|b| { - let b = b as f64 * 0.9; - b as i64 - }) - .unwrap_or(0); + let value = + query_estimated_table_rows(blockscout.connection.as_ref(), blocks::Entity.table_name()) + .await + .map_err(ChartError::BlockscoutDB)? + .map(|b| { + let b = b as f64 * 0.9; + b as i64 + }) + .unwrap_or(0); Ok(DateValue { timespan: now.date_naive(), value: value.to_string(), @@ -134,11 +94,8 @@ impl ValueEstimation for CachedBlocksEstimation { } } -pub type TotalBlocks = DirectPointLocalDbChartSourceWithEstimate< - TotalBlocksRemote, - CachedBlocksEstimation, - Properties, ->; +pub type TotalBlocks = + DirectPointLocalDbChartSourceWithEstimate; #[cfg(test)] mod tests { diff --git a/stats/stats/src/utils.rs b/stats/stats/src/utils.rs index d236ea297..d46f9b021 100644 --- a/stats/stats/src/utils.rs +++ b/stats/stats/src/utils.rs @@ -13,6 +13,7 @@ pub fn day_start(date: &NaiveDate) -> DateTime { } /// Database connection with a mark of what database it is. +/// /// Used to separate caching for different databases to /// prevent data clashes when running unit tests concurrently. #[derive(Debug, Clone)] From bdd688cb04591231502386b46dc402547a40e7bb Mon Sep 17 00:00:00 2001 From: Kirill Ivanov Date: Mon, 9 Dec 2024 16:35:24 +0900 Subject: [PATCH 39/41] set deadline for cleaning up the unused but useful part --- stats/stats/src/utils.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/stats/stats/src/utils.rs b/stats/stats/src/utils.rs index d46f9b021..203bb61ff 100644 --- a/stats/stats/src/utils.rs +++ b/stats/stats/src/utils.rs @@ -12,6 +12,9 @@ pub fn day_start(date: &NaiveDate) -> DateTime { .and_utc() } +// todo: remove marked part if not used until May 2025. +// probably rename to some wrapper of db connection to add some other +// stuff if necessary (or use UpdateContext as a place to extend the context) /// Database connection with a mark of what database it is. /// /// Used to separate caching for different databases to From ea2aab00e0562ef3fe70c433f72cd2c0ad84fdfa Mon Sep 17 00:00:00 2001 From: Kirill Ivanov Date: Wed, 11 Dec 2024 16:39:47 +0900 Subject: [PATCH 40/41] fix saturation in checked methods --- stats/stats/src/charts/types/timespans/month.rs | 10 ++++------ stats/stats/src/charts/types/timespans/week.rs | 6 ++++-- stats/stats/src/charts/types/timespans/year.rs | 4 ++-- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/stats/stats/src/charts/types/timespans/month.rs b/stats/stats/src/charts/types/timespans/month.rs index 4c7cbe867..81e35b8d8 100644 --- a/stats/stats/src/charts/types/timespans/month.rs +++ b/stats/stats/src/charts/types/timespans/month.rs @@ -77,10 +77,9 @@ impl Timespan for Month { where Self: Sized, { + let duration = chrono::Months::new(duration.repeats().try_into().ok()?); self.date_in_month - .checked_add_months(chrono::Months::new( - duration.repeats().try_into().unwrap_or(u32::MAX), - )) + .checked_add_months(duration) .map(Self::from_date) } @@ -88,10 +87,9 @@ impl Timespan for Month { where Self: Sized, { + let duration = chrono::Months::new(duration.repeats().try_into().ok()?); self.date_in_month - .checked_sub_months(chrono::Months::new( - duration.repeats().try_into().unwrap_or(u32::MAX), - )) + .checked_sub_months(duration) .map(Self::from_date) } diff --git a/stats/stats/src/charts/types/timespans/week.rs b/stats/stats/src/charts/types/timespans/week.rs index e27a7e1e8..3ac83c2e9 100644 --- a/stats/stats/src/charts/types/timespans/week.rs +++ b/stats/stats/src/charts/types/timespans/week.rs @@ -92,8 +92,9 @@ impl Timespan for Week { where Self: Sized, { + let duration = Days::new(duration.repeats().checked_mul(7)?); self.saturating_first_day() - .checked_add_days(Days::new(duration.repeats() * 7)) + .checked_add_days(duration) .map(Self::from_date) } @@ -101,8 +102,9 @@ impl Timespan for Week { where Self: Sized, { + let duration = Days::new(duration.repeats().checked_mul(7)?); self.saturating_first_day() - .checked_sub_days(Days::new(duration.repeats() * 7)) + .checked_sub_days(duration) .map(Self::from_date) } diff --git a/stats/stats/src/charts/types/timespans/year.rs b/stats/stats/src/charts/types/timespans/year.rs index 68779f1f8..67f37f92f 100644 --- a/stats/stats/src/charts/types/timespans/year.rs +++ b/stats/stats/src/charts/types/timespans/year.rs @@ -107,7 +107,7 @@ impl Timespan for Year { where Self: Sized, { - let add_years = duration.repeats().try_into().unwrap_or(i32::MAX); + let add_years = duration.repeats().try_into().ok()?; let new_year_num = self.number_within_naive_date().checked_add(add_years)?; if Self::year_number_within_naive_date(new_year_num) { Some(Self(new_year_num)) @@ -120,7 +120,7 @@ impl Timespan for Year { where Self: Sized, { - let sub_years: i32 = duration.repeats().try_into().unwrap_or(i32::MAX); + let sub_years: i32 = duration.repeats().try_into().ok()?; let new_year_num = self.number_within_naive_date().checked_sub(sub_years)?; if Self::year_number_within_naive_date(new_year_num) { Some(Self(new_year_num)) From 1ee211c303fe729644081551aad9270a05774121 Mon Sep 17 00:00:00 2001 From: Kirill Ivanov Date: Wed, 11 Dec 2024 17:16:28 +0900 Subject: [PATCH 41/41] tests --- .../stats/src/charts/types/timespans/month.rs | 34 ++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/stats/stats/src/charts/types/timespans/month.rs b/stats/stats/src/charts/types/timespans/month.rs index 81e35b8d8..baabf9dc4 100644 --- a/stats/stats/src/charts/types/timespans/month.rs +++ b/stats/stats/src/charts/types/timespans/month.rs @@ -220,7 +220,7 @@ mod tests { } #[test] - fn month_saturating_arithmetics_works() { + fn month_arithmetics_works() { assert_eq!( Month::from_date(d("2015-06-01")) .saturating_add(TimespanDuration::from_timespan_repeats(3)), @@ -231,6 +231,16 @@ mod tests { .saturating_sub(TimespanDuration::from_timespan_repeats(3)), Month::from_date(d("2015-03-01")) ); + assert_eq!( + Month::from_date(d("2015-06-01")) + .checked_add(TimespanDuration::from_timespan_repeats(3)), + Some(Month::from_date(d("2015-09-01"))) + ); + assert_eq!( + Month::from_date(d("2015-06-01")) + .checked_sub(TimespanDuration::from_timespan_repeats(3)), + Some(Month::from_date(d("2015-03-01"))) + ); assert_eq!( Month::from_date(d("2015-06-01")) @@ -243,6 +253,17 @@ mod tests { Month::from_date(NaiveDate::MIN) ); + assert_eq!( + Month::from_date(d("2015-06-01")) + .checked_add(TimespanDuration::from_timespan_repeats(u64::MAX)), + None + ); + assert_eq!( + Month::from_date(d("2015-06-01")) + .checked_sub(TimespanDuration::from_timespan_repeats(u64::MAX)), + None + ); + assert_eq!( Month::from_date(NaiveDate::MAX) .saturating_add(TimespanDuration::from_timespan_repeats(1)), @@ -253,6 +274,17 @@ mod tests { .saturating_sub(TimespanDuration::from_timespan_repeats(1)), Month::from_date(NaiveDate::MIN) ); + + assert_eq!( + Month::from_date(NaiveDate::MAX) + .checked_add(TimespanDuration::from_timespan_repeats(1)), + None + ); + assert_eq!( + Month::from_date(NaiveDate::MIN) + .checked_sub(TimespanDuration::from_timespan_repeats(1)), + None + ); } #[test]