diff --git a/stats/README.md b/stats/README.md index ca41c1695..9249a4562 100644 --- a/stats/README.md +++ b/stats/README.md @@ -92,6 +92,7 @@ by enabling word wrapping | `STATS__CONDITIONAL_​START__INTERNAL_​TRANSACTIONS_RATIO__​THRESHOLD` | | Value for `internal_​transactions_​ratio` threshold | `0.98` | | `STATS__IGNORE_​BLOCKSCOUT_API_ABSENCE` | | Disable requirement for blockscout api url setting. Turns off corresponding features if the api setting is not set | `false` | | `STATS__DISABLE_​INTERNAL_TRANSACTIONS` | | Disable functionality that utilizes internal transactions. In particular, disable internal transactions ratio check for starting the service and related charts (`newContracts`, `lastNewContracts`, and `contractsGrowth`). It has a higher priority than config files and respective envs. | `false` | +| `STATS__ENABLE_​ALL_ARBITRUM` | | Enable Arbitrum-specific charts. Variable for convenience only, the same can be done manually in configs. | `false` | [anchor]: <> (anchors.envs.end.service) diff --git a/stats/config/charts.json b/stats/config/charts.json index 2599675a3..0420f9b92 100644 --- a/stats/config/charts.json +++ b/stats/config/charts.json @@ -38,6 +38,11 @@ "title": "Total txns", "description": "All transactions including pending, dropped, replaced, failed transactions" }, + "total_operational_txns": { + "enabled": false, + "title": "Total operational txns", + "description": "'Total txns' without block creation transactions" + }, "last_new_contracts": { "title": "Number of deployed contracts today", "description": "Number of deployed contracts today" @@ -106,6 +111,16 @@ "title": "Number of transactions", "description": "Cumulative transaction growth over time" }, + "new_operational_txns": { + "enabled": false, + "title": "New operational transactions", + "description": "Number of new transactions without block creation" + }, + "operational_txns_growth": { + "enabled": false, + "title": "Number of operational transactions", + "description": "Cumulative transactions growth without block creation" + }, "txns_success_rate": { "title": "Transaction success rate", "description": "Success rate for all included transactions" diff --git a/stats/config/layout.json b/stats/config/layout.json index d55bb80f2..bc09b3a0b 100644 --- a/stats/config/layout.json +++ b/stats/config/layout.json @@ -12,7 +12,8 @@ "total_native_coin_transfers", "total_tokens", "total_txns", - "total_verified_contracts" + "total_verified_contracts", + "total_operational_txns" ], "line_chart_categories": [ { @@ -35,6 +36,8 @@ "new_txns", "txns_fee", "txns_growth", + "new_operational_txns", + "operational_txns_growth", "txns_success_rate" ] }, diff --git a/stats/config/update_groups.json b/stats/config/update_groups.json index 5016f096f..914297b9a 100644 --- a/stats/config/update_groups.json +++ b/stats/config/update_groups.json @@ -6,6 +6,7 @@ "total_addresses_group": "0 0 */3 * * * *", "total_blocks_group": "0 0 */3 * * * *", "total_tokens_group": "0 0 18 * * * *", + "total_operational_txns_group": "0 5 1 * * * *", "active_recurring_accounts_daily_recurrence_60_days_group": "0 0 2 * * * *", "active_recurring_accounts_daily_recurrence_90_days_group": "0 20 2 * * * *", "active_recurring_accounts_daily_recurrence_120_days_group": "0 40 2 * * * *", diff --git a/stats/config/utils/README.md b/stats/config/utils/README.md new file mode 100644 index 000000000..8a8b27c9f --- /dev/null +++ b/stats/config/utils/README.md @@ -0,0 +1,11 @@ +# Config utilities + +## `find_free_timeslot.py` + +It's a tool to roughly visualize the busyness of update schedule to find a timeslot for some new update group. + +### Usage + +1. Install tkinter (e.g. `apt-get install python3-tk` or `brew install python-tk`) +2. Install other dependencies from `requirements.txt`: `pip install -r requirements.txt` +3. Run `python find_free_timeslot.py` diff --git a/stats/config/utils/find_free_timeslot.py b/stats/config/utils/find_free_timeslot.py new file mode 100644 index 000000000..e6f777d54 --- /dev/null +++ b/stats/config/utils/find_free_timeslot.py @@ -0,0 +1,262 @@ +import tkinter as tk +from tkinter import ttk, filedialog, messagebox +import json +from datetime import datetime, timedelta +from croniter import croniter +import colorsys +import os +from tkcalendar import Calendar +from typing import Dict, List, Tuple + +class CronVisualizerGUI: + def __init__(self, root): + self.root = root + self.root.title("Cron Schedule Visualizer") + self.root.geometry("1200x800") + + self.schedules = {} + self.canvas_width = 1000 + self.canvas_height = 200 + self.hour_width = self.canvas_width // 24 + self.selected_date = datetime.now() + self.default_duration = 20 # Duration in minutes + + # Add default path + default_path = "../update_groups.json" + if os.path.exists(default_path): + try: + with open(default_path, 'r') as f: + data = json.load(f) + self.schedules = data.get('schedules', {}) + except Exception as e: + print(f"Failed to load default file: {str(e)}") + + self.setup_gui() + + if self.schedules: + self.update_visualization() + self.update_schedule_list() + + def setup_gui(self): + # Top frame for file selection and controls + top_frame = ttk.Frame(self.root, padding="10") + top_frame.pack(fill=tk.X) + + ttk.Button(top_frame, text="Load JSON File", command=self.load_json).pack(side=tk.LEFT, padx=5) + + self.ignore_days_var = tk.BooleanVar() + ttk.Checkbutton(top_frame, text="Ignore day parameters", + variable=self.ignore_days_var, + command=self.update_visualization).pack(side=tk.LEFT, padx=5) + + # Duration control + ttk.Label(top_frame, text="Duration (minutes):").pack(side=tk.LEFT, padx=5) + self.duration_var = tk.StringVar(value=str(self.default_duration)) + duration_entry = ttk.Entry(top_frame, textvariable=self.duration_var, width=5) + duration_entry.pack(side=tk.LEFT, padx=5) + duration_entry.bind('', lambda e: self.update_visualization()) + ttk.Button(top_frame, text="Update", command=self.update_visualization).pack(side=tk.LEFT, padx=5) + + # Calendar widget + calendar_frame = ttk.Frame(self.root, padding="10") + calendar_frame.pack(fill=tk.X) + + self.calendar = Calendar(calendar_frame, selectmode='day', + year=self.selected_date.year, + month=self.selected_date.month, + day=self.selected_date.day) + self.calendar.pack(side=tk.LEFT) + self.calendar.bind('<>', self.on_date_select) + + # Timeline canvas + canvas_frame = ttk.Frame(self.root, padding="10") + canvas_frame.pack(fill=tk.BOTH, expand=True) + + self.canvas = tk.Canvas(canvas_frame, + width=self.canvas_width, + height=self.canvas_height, + bg='white') + self.canvas.pack(fill=tk.BOTH, expand=True) + + # Bind mouse motion for hover effect + self.canvas.bind('', self.on_hover) + + # Schedule list + list_frame = ttk.Frame(self.root, padding="10") + list_frame.pack(fill=tk.BOTH, expand=True) + + self.schedule_list = ttk.Treeview(list_frame, columns=('Schedule', 'Times'), + show='headings') + self.schedule_list.heading('Schedule', text='Schedule Name') + self.schedule_list.heading('Times', text='Execution Times') + self.schedule_list.pack(fill=tk.BOTH, expand=True) + + # Status bar + self.status_var = tk.StringVar() + status_bar = ttk.Label(self.root, textvariable=self.status_var) + status_bar.pack(fill=tk.X, pady=5) + + def convert_7field_to_5field(self, cron_str: str) -> str: + """Convert 7-field cron (with seconds and years) to 5-field format.""" + fields = cron_str.split() + if len(fields) == 7: + return ' '.join(fields[1:-1]) + return cron_str + + def load_json(self): + file_path = filedialog.askopenfilename( + filetypes=[("JSON files", "*.json"), ("All files", "*.*")]) + if not file_path: + return + + try: + with open(file_path, 'r') as f: + data = json.load(f) + self.schedules = data.get('schedules', {}) + self.update_visualization() + self.update_schedule_list() + except Exception as e: + messagebox.showerror("Error", f"Failed to load file: {str(e)}") + + def get_color(self, value: int, max_value: int) -> str: + """Generate color based on value intensity.""" + if max_value == 0: + return "#FFFFFF" + + # Convert from HSV to RGB (using red hue, varying saturation) + hue = 0 # Red + saturation = min(value / max_value, 1.0) + value = 1.0 # Brightness + rgb = colorsys.hsv_to_rgb(hue, saturation, value) + + return f"#{int(rgb[0]*255):02x}{int(rgb[1]*255):02x}{int(rgb[2]*255):02x}" + + def parse_cron_schedule(self, schedule: str, target_date: datetime) -> List[datetime]: + """Parse cron schedule and return list of times it occurs in 24 hours.""" + if self.ignore_days_var.get(): + parts = schedule.split() + parts[3:] = ['*'] * len(parts[3:]) + schedule = ' '.join(parts) + + schedule = self.convert_7field_to_5field(schedule) + base = target_date.replace(hour=0, minute=0, second=0, microsecond=0) + next_day = base + timedelta(days=1) + + try: + cron = croniter(schedule, base) + times = [] + next_time = cron.get_next(datetime) + + while next_time < next_day: + times.append(next_time) + next_time = cron.get_next(datetime) + + return times + except ValueError: + return [] + + def get_task_overlaps(self) -> List[List[str]]: + """Calculate overlapping tasks for each minute of the day.""" + try: + duration = int(self.duration_var.get()) + except ValueError: + duration = self.default_duration + + # Initialize timeline with empty lists for each minute + timeline = [[] for _ in range(24 * 60)] + + # For each schedule, add its task duration to the timeline + for name, schedule in self.schedules.items(): + start_times = self.parse_cron_schedule(schedule, self.selected_date) + + for start_time in start_times: + start_minute = start_time.hour * 60 + start_time.minute + + # Add the task name to each minute it runs + for minute in range(start_minute, min(start_minute + duration, 24 * 60)): + timeline[minute].append(name) + + return timeline + + def update_visualization(self): + self.canvas.delete('all') + + # Draw hour lines and labels + for hour in range(25): + x = hour * self.hour_width + self.canvas.create_line(x, 0, x, self.canvas_height, fill='gray') + if hour < 24: + self.canvas.create_text(x + self.hour_width/2, self.canvas_height - 20, + text=f"{hour:02d}:00") + + # Get timeline with overlaps + timeline = self.get_task_overlaps() + max_overlaps = max(len(tasks) for tasks in timeline) + + # Draw visualization + for minute in range(24 * 60): + hour = minute // 60 + minute_in_hour = minute % 60 + + x = hour * self.hour_width + (minute_in_hour * self.hour_width / 60) + count = len(timeline[minute]) + + if count > 0: + color = self.get_color(count, max_overlaps) + x2 = x + self.hour_width / 60 + + self.canvas.create_rectangle( + x, 20, + x2, self.canvas_height - 40, + fill=color, outline='', + tags=('time_slot', f'minute_{minute}', + f'count_{count}', + f'tasks_{"/".join(timeline[minute])}') # Change separator to '/' + ) + + self.status_var.set(f"Maximum concurrent tasks: {max_overlaps}") + + def update_schedule_list(self): + self.schedule_list.delete(*self.schedule_list.get_children()) + for name, schedule in self.schedules.items(): + times = self.parse_cron_schedule(schedule, self.selected_date) + if times or self.ignore_days_var.get(): + time_str = ', '.join(t.strftime('%H:%M') for t in times) + self.schedule_list.insert('', 'end', values=(name, time_str)) + + def on_date_select(self, event=None): + date = self.calendar.get_date() + self.selected_date = datetime.strptime(date, '%m/%d/%y') + self.update_visualization() + self.update_schedule_list() + + def on_hover(self, event): + x, y = event.x, event.y + + if 20 <= y <= self.canvas_height - 40: + hour = int(x // self.hour_width) + minute_in_hour = int((x % self.hour_width) / (self.hour_width / 60)) + minute_index = hour * 60 + minute_in_hour + + if 0 <= minute_index < 24 * 60: + time_str = f"{hour:02d}:{minute_in_hour:02d}" + items = self.canvas.find_overlapping(x-1, 20, x+1, self.canvas_height-40) + if items: + for item in items: + tags = self.canvas.gettags(item) + # Fix 1: Check if we have a tasks tag before accessing index 3 + tasks_tag = next((tag for tag in tags if tag.startswith('tasks_')), None) + if tasks_tag: + tasks = tasks_tag[6:].split('/') # Fix 2: Change separator to '/' + count = len(tasks) + task_list = ', '.join(tasks) + self.status_var.set( + f"Time: {time_str} - {count} concurrent tasks: {task_list}") + break + else: + self.status_var.set(f"Time: {time_str} - No tasks") + +if __name__ == "__main__": + root = tk.Tk() + app = CronVisualizerGUI(root) + root.mainloop() diff --git a/stats/config/utils/requirements.txt b/stats/config/utils/requirements.txt new file mode 100644 index 000000000..b1dbf86c1 --- /dev/null +++ b/stats/config/utils/requirements.txt @@ -0,0 +1,2 @@ +tkcalendar>=1.6.1 +croniter>=2.0.3 \ No newline at end of file diff --git a/stats/justfile b/stats/justfile index cb7f40c5a..bd58f5a48 100644 --- a/stats/justfile +++ b/stats/justfile @@ -13,7 +13,7 @@ test-db-port := env_var_or_default('TEST_DB_PORT', "9433") start-postgres: # we run it in --rm mode, so all data will be deleted after stopping - docker run -p {{db-port}}:5432 --name {{docker-name}} -e POSTGRES_PASSWORD={{db-password}} -e POSTGRES_USER={{db-user}} --rm -d postgres -N 500 + docker run -p {{db-port}}:5432 --name {{docker-name}} -e POSTGRES_PASSWORD={{db-password}} -e POSTGRES_USER={{db-user}} --rm -d postgres:16 -N 500 sleep 3 # wait for postgres to start, but only if db_name is not empty $SHELL -c '[[ -z "{{db-name}}" ]] || docker exec -it {{docker-name}} psql -U postgres -c "create database {{db-name}};"' diff --git a/stats/stats-server/src/runtime_setup.rs b/stats/stats-server/src/runtime_setup.rs index bd1ba4f6d..10aeaa3cb 100644 --- a/stats/stats-server/src/runtime_setup.rs +++ b/stats/stats-server/src/runtime_setup.rs @@ -221,6 +221,7 @@ impl RuntimeSetup { Arc::new(TotalAddressesGroup), Arc::new(TotalBlocksGroup), Arc::new(TotalTokensGroup), + Arc::new(TotalOperationalTxnsGroup), Arc::new(ActiveRecurringAccountsDailyRecurrence60DaysGroup), Arc::new(ActiveRecurringAccountsMonthlyRecurrence60DaysGroup), Arc::new(ActiveRecurringAccountsWeeklyRecurrence60DaysGroup), @@ -310,6 +311,16 @@ impl RuntimeSetup { ("AverageGasPriceGroup", vec!["newTxns_DAY", "newTxns_MONTH"]), ("AverageTxnFeeGroup", vec!["newTxns_DAY", "newTxns_MONTH"]), ("TxnsSuccessRateGroup", vec!["newTxns_DAY", "newTxns_MONTH"]), + // total blocks and total txns have their own respective groups + ( + "TotalOperationalTxnsGroup", + vec!["totalBlocks_DAY", "totalTxns_DAY"], + ), + // the operational txns charts that depend on `newTxns_DAY` are + // rarely turned on, also `newTxns_DAY` is not that expensive to + // compute, therefore this solution is ok (to not introduce + // more update groups if not necessary) + ("NewBlocksGroup", vec!["newTxns_DAY"]), ] .map(|(group_name, allowed_missing)| { ( diff --git a/stats/stats-server/src/server.rs b/stats/stats-server/src/server.rs index 0bd45a44f..35c3b7450 100644 --- a/stats/stats-server/src/server.rs +++ b/stats/stats-server/src/server.rs @@ -6,7 +6,7 @@ use crate::{ health::HealthService, read_service::ReadService, runtime_setup::RuntimeSetup, - settings::{handle_disable_internal_transactions, Settings}, + settings::{handle_disable_internal_transactions, handle_enable_all_arbitrum, Settings}, update_service::UpdateService, }; @@ -66,6 +66,7 @@ pub async fn stats(mut settings: Settings) -> Result<(), anyhow::Error> { let mut charts_config = read_charts_config(&settings.charts_config)?; let layout_config = read_layout_config(&settings.layout_config)?; let update_groups_config = read_update_groups_config(&settings.update_groups_config)?; + handle_enable_all_arbitrum(settings.enable_all_arbitrum, &mut charts_config); handle_disable_internal_transactions( settings.disable_internal_transactions, &mut settings.conditional_start, diff --git a/stats/stats-server/src/settings.rs b/stats/stats-server/src/settings.rs index 3cd9ffb7a..ef5af96ad 100644 --- a/stats/stats-server/src/settings.rs +++ b/stats/stats-server/src/settings.rs @@ -8,8 +8,8 @@ use cron::Schedule; use serde::{Deserialize, Serialize}; use serde_with::{serde_as, DisplayFromStr}; use stats::{ - counters::LastNewContracts, - lines::{ContractsGrowth, NewContracts}, + counters::{LastNewContracts, TotalOperationalTxns}, + lines::{ContractsGrowth, NewContracts, NewOperationalTxns, OperationalTxnsGrowth}, ChartProperties, }; use std::{net::SocketAddr, path::PathBuf, str::FromStr}; @@ -37,6 +37,8 @@ pub struct Settings { /// /// It has a higher priority than config files and respective envs. pub disable_internal_transactions: bool, + /// Enable arbitrum-specific charts + pub enable_all_arbitrum: bool, #[serde_as(as = "DisplayFromStr")] pub default_schedule: Schedule, pub force_update_on_start: Option, // None = no update @@ -84,6 +86,7 @@ impl Default for Settings { blockscout_api_url: None, ignore_blockscout_api_absence: false, disable_internal_transactions: false, + enable_all_arbitrum: false, create_database: Default::default(), run_migrations: Default::default(), metrics: Default::default(), @@ -115,7 +118,8 @@ pub fn handle_disable_internal_transactions( warn!( "Could not disable internal transactions related chart {}: chart not found in settings. \ This should not be a problem for running the service.", - disable_key); + disable_key + ); continue; } }; @@ -124,6 +128,36 @@ pub fn handle_disable_internal_transactions( } } +pub fn handle_enable_all_arbitrum( + enable_all_arbitrum: bool, + charts: &mut config::charts::Config, +) { + if enable_all_arbitrum { + for enable_key in [ + NewOperationalTxns::key().name(), + OperationalTxnsGrowth::key().name(), + TotalOperationalTxns::key().name(), + ] { + let settings = match ( + charts.lines.get_mut(enable_key), + charts.counters.get_mut(enable_key), + ) { + (Some(settings), _) => settings, + (_, Some(settings)) => settings, + _ => { + warn!( + "Could not enable arbitrum-specific chart {}: chart not found in settings. \ + This should not be a problem for running the service.", + enable_key + ); + continue; + } + }; + settings.enabled = true; + } + } +} + /// Various limits like rate limiting and restrictions on input. #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] #[serde(default, deny_unknown_fields)] diff --git a/stats/stats-server/tests/it/lines.rs b/stats/stats-server/tests/it/lines.rs index b0f390ab1..b3cce84eb 100644 --- a/stats/stats-server/tests/it/lines.rs +++ b/stats/stats-server/tests/it/lines.rs @@ -90,6 +90,8 @@ async fn test_lines_ok() { "newTxns", "txnsFee", "txnsGrowth", + // "newOperationalTxns", + // "operationalTxnsGrowth", "txnsSuccessRate", "newVerifiedContracts", "newContracts", diff --git a/stats/stats/src/charts/counters/mod.rs b/stats/stats/src/charts/counters/mod.rs index c424d44c1..a56f22b3b 100644 --- a/stats/stats/src/charts/counters/mod.rs +++ b/stats/stats/src/charts/counters/mod.rs @@ -8,6 +8,7 @@ mod total_blocks; mod total_contracts; mod total_native_coin_holders; mod total_native_coin_transfers; +mod total_operational_txns; mod total_tokens; mod total_txns; mod total_verified_contracts; @@ -21,12 +22,13 @@ pub use last_new_contracts::LastNewContracts; pub use last_new_verified_contracts::LastNewVerifiedContracts; pub use total_accounts::TotalAccounts; pub use total_addresses::TotalAddresses; -pub use total_blocks::TotalBlocks; +pub use total_blocks::{TotalBlocks, TotalBlocksInt}; pub use total_contracts::TotalContracts; pub use total_native_coin_holders::TotalNativeCoinHolders; pub use total_native_coin_transfers::TotalNativeCoinTransfers; +pub use total_operational_txns::TotalOperationalTxns; pub use total_tokens::TotalTokens; -pub use total_txns::TotalTxns; +pub use total_txns::{TotalTxns, TotalTxnsInt}; pub use total_verified_contracts::TotalVerifiedContracts; #[cfg(test)] diff --git a/stats/stats/src/charts/counters/total_blocks.rs b/stats/stats/src/charts/counters/total_blocks.rs index 8b14fb609..762107199 100644 --- a/stats/stats/src/charts/counters/total_blocks.rs +++ b/stats/stats/src/charts/counters/total_blocks.rs @@ -3,6 +3,7 @@ use std::ops::Range; use crate::{ data_source::{ kinds::{ + data_manipulation::map::MapParseTo, local_db::DirectPointLocalDbChartSource, remote_db::{RemoteDatabaseSource, RemoteQueryBehaviour}, }, @@ -73,6 +74,7 @@ impl ChartProperties for Properties { } pub type TotalBlocks = DirectPointLocalDbChartSource; +pub type TotalBlocksInt = MapParseTo; #[cfg(test)] mod tests { diff --git a/stats/stats/src/charts/counters/total_operational_txns.rs b/stats/stats/src/charts/counters/total_operational_txns.rs new file mode 100644 index 000000000..a936c0c9e --- /dev/null +++ b/stats/stats/src/charts/counters/total_operational_txns.rs @@ -0,0 +1,80 @@ +use crate::{ + data_source::{ + kinds::{ + data_manipulation::map::{Map, MapFunction}, + local_db::DirectPointLocalDbChartSource, + }, + DataSource, + }, + types::TimespanValue, + ChartProperties, MissingDatePolicy, Named, +}; + +use chrono::NaiveDate; +use entity::sea_orm_active_enums::ChartType; +use tracing::warn; + +use super::{TotalBlocksInt, TotalTxnsInt}; + +pub struct Properties; + +impl Named for Properties { + fn name() -> String { + "totalOperationalTxns".into() + } +} + +impl ChartProperties for Properties { + type Resolution = NaiveDate; + + fn chart_type() -> ChartType { + ChartType::Counter + } + + fn missing_date_policy() -> MissingDatePolicy { + MissingDatePolicy::FillPrevious + } +} + +pub struct Calculate; + +type Input = ( + ::Output, + ::Output, +); + +impl MapFunction for Calculate { + type Output = TimespanValue; + + fn function(inner_data: Input) -> Result { + let (total_blocks_data, total_txns_data) = inner_data; + if total_blocks_data.timespan != total_txns_data.timespan { + warn!("timespans for total blocks and total transactions do not match when calculating {}", Properties::name()); + } + let date = total_blocks_data.timespan; + let value = total_txns_data + .value + .saturating_sub(total_blocks_data.value); + Ok(TimespanValue { + timespan: date, + value: value.to_string(), + }) + } +} + +pub type TotalOperationalTxns = + DirectPointLocalDbChartSource, Properties>; + +#[cfg(test)] +mod tests { + use super::*; + use crate::tests::simple_test::simple_test_counter; + + #[tokio::test] + #[ignore = "needs database to run"] + async fn update_total_operational_txns() { + // 47 - 13 (txns - blocks) + simple_test_counter::("update_total_operational_txns", "34", None) + .await; + } +} diff --git a/stats/stats/src/charts/counters/total_txns.rs b/stats/stats/src/charts/counters/total_txns.rs index 33bcc40a0..af75e1f76 100644 --- a/stats/stats/src/charts/counters/total_txns.rs +++ b/stats/stats/src/charts/counters/total_txns.rs @@ -1,6 +1,9 @@ use crate::{ data_source::kinds::{ - data_manipulation::{map::MapToString, sum_point::Sum}, + data_manipulation::{ + map::{MapParseTo, MapToString}, + sum_point::Sum, + }, local_db::DirectPointLocalDbChartSource, }, lines::NewTxnsInt, @@ -30,6 +33,7 @@ impl ChartProperties for Properties { } pub type TotalTxns = DirectPointLocalDbChartSource>, Properties>; +pub type TotalTxnsInt = MapParseTo; #[cfg(test)] mod tests { diff --git a/stats/stats/src/charts/lines/mod.rs b/stats/stats/src/charts/lines/mod.rs index 7b3fa5a5d..3eff4dcb7 100644 --- a/stats/stats/src/charts/lines/mod.rs +++ b/stats/stats/src/charts/lines/mod.rs @@ -16,8 +16,10 @@ mod new_blocks; mod new_contracts; mod new_native_coin_holders; mod new_native_coin_transfers; +mod new_operational_txns; mod new_txns; mod new_verified_contracts; +mod operational_txns_growth; mod txns_fee; mod txns_growth; mod txns_success_rate; @@ -88,11 +90,19 @@ pub use new_native_coin_transfers::{ NewNativeCoinTransfers, NewNativeCoinTransfersInt, NewNativeCoinTransfersMonthly, NewNativeCoinTransfersWeekly, NewNativeCoinTransfersYearly, }; +pub use new_operational_txns::{ + NewOperationalTxns, NewOperationalTxnsMonthly, NewOperationalTxnsWeekly, + NewOperationalTxnsYearly, +}; pub use new_txns::{NewTxns, NewTxnsInt, NewTxnsMonthly, NewTxnsWeekly, NewTxnsYearly}; pub use new_verified_contracts::{ NewVerifiedContracts, NewVerifiedContractsMonthly, NewVerifiedContractsWeekly, NewVerifiedContractsYearly, }; +pub use operational_txns_growth::{ + OperationalTxnsGrowth, OperationalTxnsGrowthMonthly, OperationalTxnsGrowthWeekly, + OperationalTxnsGrowthYearly, +}; pub use txns_fee::{TxnsFee, TxnsFeeMonthly, TxnsFeeWeekly, TxnsFeeYearly}; pub use txns_growth::{TxnsGrowth, TxnsGrowthMonthly, TxnsGrowthWeekly, TxnsGrowthYearly}; pub use txns_success_rate::{ diff --git a/stats/stats/src/charts/lines/new_operational_txns.rs b/stats/stats/src/charts/lines/new_operational_txns.rs new file mode 100644 index 000000000..9162cb05e --- /dev/null +++ b/stats/stats/src/charts/lines/new_operational_txns.rs @@ -0,0 +1,141 @@ +use crate::{ + data_processing::zip_same_timespan, + data_source::kinds::{ + data_manipulation::{ + map::{Map, MapFunction, MapParseTo, MapToString}, + resolutions::sum::SumLowerResolution, + }, + local_db::{ + parameters::update::batching::parameters::{ + Batch30Weeks, Batch30Years, Batch36Months, BatchMaxDays, + }, + DirectVecLocalDbChartSource, + }, + }, + define_and_impl_resolution_properties, + types::{ + timespans::{Month, Week, Year}, + Timespan, TimespanValue, + }, + ChartProperties, MissingDatePolicy, Named, +}; + +use chrono::NaiveDate; +use entity::sea_orm_active_enums::ChartType; +use itertools::Itertools; + +use super::{new_blocks::NewBlocksInt, NewTxnsInt}; + +pub struct Properties; + +impl Named for Properties { + fn name() -> String { + "newOperationalTxns".into() + } +} + +impl ChartProperties for Properties { + type Resolution = NaiveDate; + + fn chart_type() -> ChartType { + ChartType::Line + } + + fn missing_date_policy() -> MissingDatePolicy { + MissingDatePolicy::FillZero + } +} + +pub struct Calculate; + +type Input = ( + // newBlocks + Vec>, + // newTxns + Vec>, +); + +impl MapFunction> for Calculate +where + Resolution: Timespan + Send + Ord, +{ + type Output = Vec>; + + fn function(inner_data: Input) -> Result { + let (blocks_data, txns_data) = inner_data; + let combined = zip_same_timespan(blocks_data, txns_data); + let data = combined + .into_iter() + .map(|(timespan, data)| { + // both new blocks & new txns treat missing values as zero + let (blocks, txns) = data.or(0, 0); + TimespanValue { + timespan, + value: txns.saturating_sub(blocks).to_string(), + } + }) + .collect_vec(); + Ok(data) + } +} + +define_and_impl_resolution_properties!( + define_and_impl: { + WeeklyProperties: Week, + MonthlyProperties: Month, + YearlyProperties: Year, + }, + base_impl: Properties +); + +pub type NewOperationalTxns = DirectVecLocalDbChartSource< + Map<(NewBlocksInt, NewTxnsInt), Calculate>, + BatchMaxDays, + Properties, +>; +pub type NewOperationalTxnsInt = MapParseTo; +pub type NewOperationalTxnsWeekly = DirectVecLocalDbChartSource< + MapToString>, + Batch30Weeks, + WeeklyProperties, +>; +pub type NewOperationalTxnsMonthly = DirectVecLocalDbChartSource< + MapToString>, + Batch36Months, + MonthlyProperties, +>; +pub type NewOperationalTxnsMonthlyInt = MapParseTo; +pub type NewOperationalTxnsYearly = DirectVecLocalDbChartSource< + MapToString>, + Batch30Years, + YearlyProperties, +>; + +#[cfg(test)] +mod tests { + use super::*; + use crate::tests::simple_test::simple_test_chart; + + #[tokio::test] + #[ignore = "needs database to run"] + async fn update_new_operational_txns() { + simple_test_chart::( + "update_new_operational_txns", + vec![ + ("2022-11-09", "4"), + ("2022-11-10", "9"), + ("2022-11-11", "10"), + ("2022-11-12", "4"), + ("2022-12-01", "4"), + ("2023-01-01", "0"), + ("2023-02-01", "3"), + ("2023-03-01", "0"), + ], + ) + .await; + } + + // the implementation is generic over resolutions, + // therefore other res should also work fine + // (tests are becoming excruciatingly slow) +} diff --git a/stats/stats/src/charts/lines/operational_txns_growth.rs b/stats/stats/src/charts/lines/operational_txns_growth.rs new file mode 100644 index 000000000..9c847f061 --- /dev/null +++ b/stats/stats/src/charts/lines/operational_txns_growth.rs @@ -0,0 +1,98 @@ +use crate::{ + data_source::kinds::{ + data_manipulation::{ + map::{MapParseTo, MapToString}, + resolutions::last_value::LastValueLowerResolution, + }, + local_db::{ + parameters::update::batching::parameters::{Batch30Weeks, Batch30Years, Batch36Months}, + DailyCumulativeLocalDbChartSource, DirectVecLocalDbChartSource, + }, + }, + define_and_impl_resolution_properties, + types::timespans::{Month, Week, Year}, + ChartProperties, MissingDatePolicy, Named, +}; + +use chrono::NaiveDate; +use entity::sea_orm_active_enums::ChartType; + +use super::new_operational_txns::NewOperationalTxns; + +pub struct Properties; + +impl Named for Properties { + fn name() -> String { + "operationalTxnsGrowth".into() + } +} + +impl ChartProperties for Properties { + type Resolution = NaiveDate; + + fn chart_type() -> ChartType { + ChartType::Line + } + + fn missing_date_policy() -> MissingDatePolicy { + MissingDatePolicy::FillPrevious + } +} + +define_and_impl_resolution_properties!( + define_and_impl: { + WeeklyProperties: Week, + MonthlyProperties: Month, + YearlyProperties: Year, + }, + base_impl: Properties +); + +pub type OperationalTxnsGrowth = + DailyCumulativeLocalDbChartSource, Properties>; +pub type OperationalTxnsGrowthInt = MapParseTo; +pub type OperationalTxnsGrowthWeekly = DirectVecLocalDbChartSource< + MapToString>, + Batch30Weeks, + WeeklyProperties, +>; +pub type OperationalTxnsGrowthMonthly = DirectVecLocalDbChartSource< + MapToString>, + Batch36Months, + MonthlyProperties, +>; +pub type OperationalTxnsGrowthMonthlyInt = MapParseTo; +pub type OperationalTxnsGrowthYearly = DirectVecLocalDbChartSource< + MapToString>, + Batch30Years, + YearlyProperties, +>; + +#[cfg(test)] +mod tests { + use super::*; + use crate::tests::simple_test::simple_test_chart; + + #[tokio::test] + #[ignore = "needs database to run"] + async fn update_operational_txns_growth() { + simple_test_chart::( + "update_operational_txns_growth", + vec![ + ("2022-11-09", "4"), + ("2022-11-10", "13"), + ("2022-11-11", "23"), + ("2022-11-12", "27"), + ("2022-12-01", "31"), + ("2023-01-01", "31"), + ("2023-02-01", "34"), + ("2023-03-01", "34"), + ], + ) + .await; + } + + // the implementation is generic over resolutions, + // therefore other res should also work fine + // (tests are becoming excruciatingly slow) +} diff --git a/stats/stats/src/data_processing.rs b/stats/stats/src/data_processing.rs index 99195854a..d23d1d65c 100644 --- a/stats/stats/src/data_processing.rs +++ b/stats/stats/src/data_processing.rs @@ -1,4 +1,5 @@ use chrono::NaiveDate; +use itertools::EitherOrBoth; use crate::{ charts::types::timespans::DateValue, @@ -6,6 +7,7 @@ use crate::{ UpdateError, }; use std::{ + cmp::Ordering, mem, ops::{AddAssign, SubAssign}, }; @@ -83,13 +85,67 @@ pub fn last_point(data: Vec>) -> Option> { data.into_iter().max() } +/// "zip" two sorted date/value vectors, combining +/// values with the same date. +/// +/// If both vectors contain values for a date, it yields two values via `EitherOrBoth::Both`. +/// +/// If only one of the vectors contains a value for a date, it yields the value via `EitherOrBoth::Left` +/// or `EitherOrBoth::Right`. +pub fn zip_same_timespan( + left: Vec>, + right: Vec>, +) -> Vec<(T, EitherOrBoth)> +where + T: Ord, +{ + let mut left = left.into_iter().peekable(); + let mut right = right.into_iter().peekable(); + let mut result = vec![]; + loop { + match (left.peek(), right.peek()) { + (Some(l), Some(r)) => { + let (left_t, right_t) = (&l.timespan, &r.timespan); + match left_t.cmp(right_t) { + Ordering::Equal => { + let (l, r) = ( + left.next().expect("peek just succeeded"), + right.next().expect("peek just succeeded"), + ); + result.push((l.timespan, EitherOrBoth::Both(l.value, r.value))) + } + Ordering::Less => { + let left_point = left.next().expect("peek just succeeded"); + result.push((left_point.timespan, EitherOrBoth::Left(left_point.value))) + } + Ordering::Greater => { + let right_point = right.next().expect("peek just succeeded"); + result.push((right_point.timespan, EitherOrBoth::Right(right_point.value))) + } + } + } + (Some(_), None) => { + result.extend(left.map(|p| (p.timespan, EitherOrBoth::Left(p.value)))); + break; + } + (None, Some(_)) => { + result.extend(right.map(|p| (p.timespan, EitherOrBoth::Right(p.value)))); + break; + } + (None, None) => break, + } + } + result +} + #[cfg(test)] mod tests { + use itertools::EitherOrBoth; use pretty_assertions::assert_eq; use rust_decimal_macros::dec; use super::*; - use crate::tests::point_construction::{d_v_decimal, d_v_int}; + use crate::tests::point_construction::{d, d_v, d_v_decimal, d_v_int}; #[test] fn test_deltas_works_int() { @@ -306,4 +362,72 @@ mod tests { assert_eq!(cumsum(data, initial_value).unwrap(), expected); } } + + #[test] + fn zip_same_timespan_works() { + assert_eq!( + zip_same_timespan::(vec![], vec![]), + vec![] + ); + assert_eq!( + zip_same_timespan::( + vec![], + vec![ + d_v("2024-07-05", "5R"), + d_v("2024-07-07", "7R"), + d_v("2024-07-08", "8R"), + d_v("2024-07-11", "11R"), + ] + ), + vec![ + (d("2024-07-05"), EitherOrBoth::Right("5R".to_string())), + (d("2024-07-07"), EitherOrBoth::Right("7R".to_string())), + (d("2024-07-08"), EitherOrBoth::Right("8R".to_string())), + (d("2024-07-11"), EitherOrBoth::Right("11R".to_string())), + ] + ); + assert_eq!( + zip_same_timespan::( + vec![ + d_v("2024-07-05", "5L"), + d_v("2024-07-07", "7L"), + d_v("2024-07-08", "8L"), + d_v("2024-07-11", "11L"), + ], + vec![] + ), + vec![ + (d("2024-07-05"), EitherOrBoth::Left("5L".to_string())), + (d("2024-07-07"), EitherOrBoth::Left("7L".to_string())), + (d("2024-07-08"), EitherOrBoth::Left("8L".to_string())), + (d("2024-07-11"), EitherOrBoth::Left("11L".to_string())), + ] + ); + assert_eq!( + zip_same_timespan( + vec![ + d_v("2024-07-08", "8L"), + d_v("2024-07-09", "9L"), + d_v("2024-07-10", "10L"), + ], + vec![ + d_v("2024-07-05", "5R"), + d_v("2024-07-07", "7R"), + d_v("2024-07-08", "8R"), + d_v("2024-07-11", "11R"), + ] + ), + vec![ + (d("2024-07-05"), EitherOrBoth::Right("5R".to_string())), + (d("2024-07-07"), EitherOrBoth::Right("7R".to_string())), + ( + d("2024-07-08"), + EitherOrBoth::Both("8L".to_string(), "8R".to_string()) + ), + (d("2024-07-09"), EitherOrBoth::Left("9L".to_string())), + (d("2024-07-10"), EitherOrBoth::Left("10L".to_string())), + (d("2024-07-11"), EitherOrBoth::Right("11R".to_string())), + ] + ) + } } diff --git a/stats/stats/src/data_source/kinds/data_manipulation/resolutions/average.rs b/stats/stats/src/data_source/kinds/data_manipulation/resolutions/average.rs index 7bc8dc642..b9c2ec6e8 100644 --- a/stats/stats/src/data_source/kinds/data_manipulation/resolutions/average.rs +++ b/stats/stats/src/data_source/kinds/data_manipulation/resolutions/average.rs @@ -1,5 +1,5 @@ //! Constructors for lower resolutions of average value charts -use std::{cmp::Ordering, fmt::Debug, marker::PhantomData, ops::Range}; +use std::{fmt::Debug, marker::PhantomData, ops::Range}; use blockscout_metrics_tools::AggregateTimer; use chrono::{DateTime, Utc}; @@ -7,6 +7,7 @@ use itertools::{EitherOrBoth, Itertools}; use sea_orm::{prelude::DateTimeUtc, DatabaseConnection, DbErr}; use crate::{ + data_processing::zip_same_timespan, data_source::{ kinds::data_manipulation::resolutions::reduce_each_timespan, DataSource, UpdateContext, }, @@ -77,59 +78,6 @@ where } } -/// "zip" two sorted date/value vectors, combining -/// values with the same date. -/// -/// If both vectors contain values for a date, it yields two values via `EitherOrBoth::Both`. -/// -/// If only one of the vectors contains a value for a date, it yields the value via `EitherOrBoth::Left` -/// or `EitherOrBoth::Right`. -fn zip_same_timespan( - left: Vec>, - right: Vec>, -) -> Vec<(T, EitherOrBoth)> -where - T: Ord, -{ - let mut left = left.into_iter().peekable(); - let mut right = right.into_iter().peekable(); - let mut result = vec![]; - loop { - match (left.peek(), right.peek()) { - (Some(l), Some(r)) => { - let (left_t, right_t) = (&l.timespan, &r.timespan); - match left_t.cmp(right_t) { - Ordering::Equal => { - let (l, r) = ( - left.next().expect("peek just succeeded"), - right.next().expect("peek just succeeded"), - ); - result.push((l.timespan, EitherOrBoth::Both(l.value, r.value))) - } - Ordering::Less => { - let left_point = left.next().expect("peek just succeeded"); - result.push((left_point.timespan, EitherOrBoth::Left(left_point.value))) - } - Ordering::Greater => { - let right_point = right.next().expect("peek just succeeded"); - result.push((right_point.timespan, EitherOrBoth::Right(right_point.value))) - } - } - } - (Some(_), None) => { - result.extend(left.map(|p| (p.timespan, EitherOrBoth::Left(p.value)))); - break; - } - (None, Some(_)) => { - result.extend(right.map(|p| (p.timespan, EitherOrBoth::Right(p.value)))); - break; - } - (None, None) => break, - } - } - result -} - fn lower_res_average_from( h_res_average: Vec>, h_res_weight: Vec>, @@ -192,7 +140,7 @@ mod tests { data_source::{kinds::data_manipulation::map::MapParseTo, types::BlockscoutMigrations}, gettable_const, lines::{PredefinedMockSource, PseudoRandomMockRetrieve}, - tests::point_construction::{d, d_v, d_v_double, d_v_int, dt, w_v_double, week_of}, + tests::point_construction::{d, d_v_double, d_v_int, dt, w_v_double, week_of}, types::timespans::{DateValue, Week, WeekValue}, MissingDatePolicy, }; @@ -203,74 +151,6 @@ mod tests { use itertools::Itertools; use pretty_assertions::assert_eq; - #[test] - fn zip_same_timespan_works() { - assert_eq!( - zip_same_timespan::(vec![], vec![]), - vec![] - ); - assert_eq!( - zip_same_timespan::( - vec![], - vec![ - d_v("2024-07-05", "5R"), - d_v("2024-07-07", "7R"), - d_v("2024-07-08", "8R"), - d_v("2024-07-11", "11R"), - ] - ), - vec![ - (d("2024-07-05"), EitherOrBoth::Right("5R".to_string())), - (d("2024-07-07"), EitherOrBoth::Right("7R".to_string())), - (d("2024-07-08"), EitherOrBoth::Right("8R".to_string())), - (d("2024-07-11"), EitherOrBoth::Right("11R".to_string())), - ] - ); - assert_eq!( - zip_same_timespan::( - vec![ - d_v("2024-07-05", "5L"), - d_v("2024-07-07", "7L"), - d_v("2024-07-08", "8L"), - d_v("2024-07-11", "11L"), - ], - vec![] - ), - vec![ - (d("2024-07-05"), EitherOrBoth::Left("5L".to_string())), - (d("2024-07-07"), EitherOrBoth::Left("7L".to_string())), - (d("2024-07-08"), EitherOrBoth::Left("8L".to_string())), - (d("2024-07-11"), EitherOrBoth::Left("11L".to_string())), - ] - ); - assert_eq!( - zip_same_timespan( - vec![ - d_v("2024-07-08", "8L"), - d_v("2024-07-09", "9L"), - d_v("2024-07-10", "10L"), - ], - vec![ - d_v("2024-07-05", "5R"), - d_v("2024-07-07", "7R"), - d_v("2024-07-08", "8R"), - d_v("2024-07-11", "11R"), - ] - ), - vec![ - (d("2024-07-05"), EitherOrBoth::Right("5R".to_string())), - (d("2024-07-07"), EitherOrBoth::Right("7R".to_string())), - ( - d("2024-07-08"), - EitherOrBoth::Both("8L".to_string(), "8R".to_string()) - ), - (d("2024-07-09"), EitherOrBoth::Left("9L".to_string())), - (d("2024-07-10"), EitherOrBoth::Left("10L".to_string())), - (d("2024-07-11"), EitherOrBoth::Right("11R".to_string())), - ] - ) - } - #[test] fn weekly_average_from_works() { // weeks for this month are diff --git a/stats/stats/src/lib.rs b/stats/stats/src/lib.rs index 6fd73cc4a..bd4fe56e8 100644 --- a/stats/stats/src/lib.rs +++ b/stats/stats/src/lib.rs @@ -1,3 +1,5 @@ +#![recursion_limit = "256"] + mod charts; pub mod data_processing; pub mod data_source; diff --git a/stats/stats/src/update_groups.rs b/stats/stats/src/update_groups.rs index 278beeb81..e7afa83af 100644 --- a/stats/stats/src/update_groups.rs +++ b/stats/stats/src/update_groups.rs @@ -23,6 +23,10 @@ singleton_groups!( TotalAddresses, TotalBlocks, TotalTokens, + // Even though it depends on `TotalTxns` and `TotalBlocks`, + // it's ok not to update it as frequently. + // Granular control over these 2 still seems useful. + TotalOperationalTxns, // Each of the `ActiveRecurringAccounts*` charts includes quite heavy SQL query, // thus it's better to have granular control on update times. ActiveRecurringAccountsDailyRecurrence60Days, @@ -108,6 +112,15 @@ construct_update_group!(NewBlocksGroup { NewBlocksWeekly, NewBlocksMonthly, NewBlocksYearly, + // if the following are enabled, then NewTxns is updated as well + NewOperationalTxns, + NewOperationalTxnsWeekly, + NewOperationalTxnsMonthly, + NewOperationalTxnsYearly, + OperationalTxnsGrowth, + OperationalTxnsGrowthWeekly, + OperationalTxnsGrowthMonthly, + OperationalTxnsGrowthYearly ] });