diff --git a/.gitignore b/.gitignore index bcbfe4c5c5..488dae5995 100644 --- a/.gitignore +++ b/.gitignore @@ -424,4 +424,5 @@ leak_report temp/ logs.db +benchmarks.html agent/ \ No newline at end of file diff --git a/Pipfile b/Pipfile index b697c6bfdf..ad9b7b51d4 100644 --- a/Pipfile +++ b/Pipfile @@ -38,9 +38,6 @@ open-aea-cosmpy = "==0.6.5" grpcio = "==1.53.0" hypothesis = "==6.21.6" # latest supported for Python 3.7 -numpy = ">=1.21.6" -pandas = "==1.5.3" -pandas-stubs = "==1.2.0.62" protobuf = "<=3.20.1,>=3.19" pytz = "==2022.2.1" py-ecc = "==6.0.0" diff --git a/autonomy/analyse/benchmark/aggregate.py b/autonomy/analyse/benchmark/aggregate.py index d2c51204e8..7fad25af28 100644 --- a/autonomy/analyse/benchmark/aggregate.py +++ b/autonomy/analyse/benchmark/aggregate.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2021-2022 Valory AG +# Copyright 2021-2023 Valory AG # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -20,12 +20,25 @@ """Tools for aggregating benchmark results.""" import json +import statistics from pathlib import Path -from typing import Any, Dict, List +from typing import Callable, Dict, List, Tuple, cast -import pandas as pd +from autonomy.analyse.benchmark.html import ( + BLOCK_TEMPLATE, + HTML_TEMPLATE, + TABLE_TEMPLATE, + TD_TEMPLATE, + TH_TEMPLATE, + TROW_TEMPLATE, +) -from autonomy.analyse.benchmark.html import HTML_TEMPLATE, TABLE_TEMPLATE + +STATISTICS = { + "Mean": statistics.mean, + "Maximum": max, + "Minimum": min, +} class BlockTypes: # pylint: disable=too-few-public-methods @@ -39,95 +52,149 @@ class BlockTypes: # pylint: disable=too-few-public-methods types = (LOCAL, CONSENSUS, TOTAL) -def read_benchmark_data(path: Path) -> List[Dict]: +def read_benchmark_data( + path: Path, + block_type: str = BlockTypes.ALL, + period: int = -1, +) -> Dict[str, Dict[str, List[Dict]]]: """Returns logs.""" - benchmark_data = [] + benchmark_data: Dict[str, Dict[str, List[Dict]]] = {} for agent_dir in path.iterdir(): - agent_benchmark_data: Dict[str, Any] = {} - agent_benchmark_data["name"] = agent_dir.name - agent_benchmark_data["data"] = dict( - map( - lambda path: ( - int(path.name.replace(".json", "")), - json.loads(path.read_text()), - ), - agent_dir.glob("*.json"), - ) + benchmark_data[agent_dir.name] = {} + benchmark_data_files = list(agent_dir.glob("*.json")) + periods = sorted( + tuple(map(lambda x: x.name, benchmark_data_files)) + if period == -1 + else (f"{period}.json",) ) - benchmark_data.append(agent_benchmark_data) - return benchmark_data - - -def create_dataframe(data: List[Dict]) -> pd.DataFrame: - """Create pandas.DataFrame object from benchmark data.""" - - rows = [] - behaviours = [behaviour_data["behaviour"] for behaviour_data in data[0]["data"][0]] - cols = ["agent", "period", "block_type", *behaviours] - - for agent in data: - for period_n, period_data in agent["data"].items(): - for block_t in BlockTypes.types: - rows.append( - { - "agent": agent["name"], - "period": period_n, - "block_type": block_t, - **{ - behaviour_data["behaviour"]: behaviour_data["data"].get( - block_t, 0.0 - ) - for behaviour_data in period_data - }, - } - ) + for _period in periods: + period_name, _ = _period.split(".") + period_data = json.loads((agent_dir / _period).read_text()) + if block_type == BlockTypes.ALL: + benchmark_data[agent_dir.name][period_name] = period_data + continue + + benchmark_data[agent_dir.name][period_name] = [ + { + "behaviour": behaviour["behaviour"], + "data": {block_type: behaviour["data"][block_type]}, + } + for behaviour in period_data + ] - return pd.DataFrame( - data=rows, - )[cols] - - -def format_output(df: pd.DataFrame, period: int, block_type: str) -> str: - """Format output from given dataframe and parameters""" - - df = df.copy(deep=True).fillna(value=0.0) - df = df[df["period"] == period] - df = df[df["block_type"] == block_type] + return benchmark_data - del df["period"] - del df["block_type"] - time_df = df[df.columns[1:]] - stats_df = pd.DataFrame( - data=[ - ["mean", *time_df.mean().values], - ["median", *time_df.median().values], - ["std_dev", *time_df.std().values], - ], - columns=df.columns, +def add_statistic( + name: str, + aggregator: Callable, + behaviours: List[str], + behaviour_history: Dict[str, List[float]], +) -> str: + """Add a stastic column.""" + rows = TD_TEMPLATE.format(name) + for behaviour in behaviours: + rows += TD_TEMPLATE.format(aggregator(behaviour_history[behaviour])) + return TROW_TEMPLATE.format(rows) + + +def add_statistics( + behaviours: List[str], + behaviour_history: Dict[str, List[float]], +) -> str: + """Add statistics.""" + tbody = TROW_TEMPLATE.format( + f"""