From e2af9bec1300a36ec5fd696347b8bfbe5d01407f Mon Sep 17 00:00:00 2001 From: Cortze Date: Mon, 17 May 2021 09:39:25 +0200 Subject: [PATCH 1/5] Fix Error types titles --- src/analyzer/armiarma-analyzer.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/analyzer/armiarma-analyzer.py b/src/analyzer/armiarma-analyzer.py index 757419e..94df68a 100644 --- a/src/analyzer/armiarma-analyzer.py +++ b/src/analyzer/armiarma-analyzer.py @@ -755,15 +755,15 @@ def main(): 'textSize': textSize+2, 'yLowLimit': None, 'yUpperLimit': None, - 'title': "Distribution of the detected errors", - 'xlabel': 'Number of peers', + 'title': "Distribution of the Experienced Connection Errors", + 'xlabel': 'Peers With Same Connection Error', 'ylabel': None, 'yticks': None, 'vGrids': True, 'titleSize': titleSize+2, 'labelSize': labelSize+2, 'legendPosition': 1, - 'legendTitle': 'Experienced errors', + 'legendTitle': 'Recorded errors', 'legendSize': labelSize-4, 'xticksSize': ticksSize, 'yticksSize': ticksSize+2, From a9f922e04638483a2aa7ff0681474e77d6ab0965 Mon Sep 17 00:00:00 2001 From: Cortze Date: Mon, 17 May 2021 12:29:54 +0200 Subject: [PATCH 2/5] Add script to analyze the global overview of the client distribution --- src/analyzer/total-overview-analysis.py | 100 ++++++++++++++++++++++++ 1 file changed, 100 insertions(+) create mode 100644 src/analyzer/total-overview-analysis.py diff --git a/src/analyzer/total-overview-analysis.py b/src/analyzer/total-overview-analysis.py new file mode 100644 index 0000000..5e30ea1 --- /dev/null +++ b/src/analyzer/total-overview-analysis.py @@ -0,0 +1,100 @@ +# This file compiles the code to analyze the gathered metrics from each of the +# folders in the $ARMIARMAPATH/examples folder, providing a global overview of the +# Peer distributions and client types over the dayly analysis + +import os, sys +import json +import pandas as pd +import matplotlib.pyplot as plt + + +def mainexecution(): + projectsFolder = sys.argv[1] + + # So far, the generated panda will just contain the client count + p = {'date': [], 'Lighthouse': [], 'Teku': [], 'Nimbus': [], 'Prysm': [], 'Lodestar': [], 'Unknown': []} + print(projectsFolder) + for root, dirs, _ in os.walk(projectsFolder): + print(dirs) + # We just need to access the folders inside examples + for d in dirs: + fold = os.path.join(root, d) + f = fold + '/metrics/custom-metrics.json' + if os.path.exists(f): + # Load the readed values from the json into the panda + poblatePandaCustomMetrics(p, f) + break + # After filling the dict with all the info from the JSONs, generate the panda + df = pd.DataFrame (p, columns = ['date', 'Lighthouse', 'Teku', 'Nimbus', 'Prysm', 'Lodestar', 'Unknown']) + print(df) + + clientList = ['Lighthouse', 'Teku', 'Nimbus', 'Prysm', 'Lodestar', 'Unknown'] + figSize = (10,6) + titleSize = 24 + textSize = 20 + labelSize = 20 + # Plot the images + plotStackedChart(df, opts={ + 'figSize': figSize, + 'figTitle': 'CrawlSummary.png', + 'align': 'center', + 'textSize': textSize, + 'title': "Evolution of the observed client distribution", + 'ylabel': 'Client concentration (%)', + 'xlabel': None, + 'legendLabels': clientList, + 'titleSize': titleSize, + 'labelSize': labelSize, + 'lengendPosition': 1, + 'legendSize': labelSize, + 'tickRotation': 90, + 'show': True}) + +def poblatePandaCustomMetrics(p, jsonFile): + print('poblating panda') + # Read the Json + jsf = open(jsonFile) + jsonValues = json.load(jsf) + jsf.close() + + # Compose the date of the crawling day + cday = str(jsonValues['StartTime']['Year']) + '-' + str(jsonValues['StartTime']['Month']) + '-' + str(jsonValues['StartTime']['Day']) + p['date'].append(cday) + ps = jsonValues['PeerStore']['ConnectionSucceed']['Total'] + # Get percentages for each of the clients + lig = (100 * int(jsonValues['PeerStore']['ConnectionSucceed']['Lighthouse']['Total'])) / ps + tek = (100 * int(jsonValues['PeerStore']['ConnectionSucceed']['Teku']['Total'])) / ps + nim = (100 * int(jsonValues['PeerStore']['ConnectionSucceed']['Nimbus']['Total'])) / ps + pry = (100 * int(jsonValues['PeerStore']['ConnectionSucceed']['Prysm']['Total'])) / ps + lod = (100 * int(jsonValues['PeerStore']['ConnectionSucceed']['Lodestar']['Total'])) / ps + unk = (100 * int(jsonValues['PeerStore']['ConnectionSucceed']['Unknown']['Total'])) / ps + + p['Lighthouse'].append(lig) + p['Teku'].append(tek) + p['Nimbus'].append(nim) + p['Prysm'].append(pry) + p['Lodestar'].append(lod) + p['Unknown'].append(unk) + + +def plotStackedChart(p, opts): + + ax = p.plot.area(figsize = opts['figSize'], x='date', stacked=True) + + # labels + if opts['ylabel'] is not None: + plt.ylabel(opts['ylabel'], fontsize=opts['labelSize']) + if opts['xlabel'] is not None: + plt.xlabel(opts['xlabel'], fontsize=opts['labelSize']) + + handles,legends = ax.get_legend_handles_labels() + ax.legend(handles=handles, labels=opts['legendLabels'], loc='upper center', bbox_to_anchor=(0.5, -0.05), fancybox=True, shadow=True, ncol=6) + + plt.title(opts['title'], fontsize = opts['titleSize']) + plt.tight_layout() + #plt.savefig(outputFile) + if opts['show'] is True: + plt.show() + +if __name__ == '__main__': + mainexecution() \ No newline at end of file From 15c29260613fdc72f833cd3def1b300bce3f41d1 Mon Sep 17 00:00:00 2001 From: Mikel Cortes Date: Mon, 17 May 2021 18:15:08 +0200 Subject: [PATCH 3/5] Add overall view analyzer as little mods in analyzer --- .gitignore | 6 +-- src/analyzer/armiarma-analyzer.py | 2 +- src/analyzer/total-overview-analysis.py | 72 +++++++++++++++++++------ 3 files changed, 61 insertions(+), 19 deletions(-) diff --git a/.gitignore b/.gitignore index c37f12d..651d89c 100644 --- a/.gitignore +++ b/.gitignore @@ -8,12 +8,12 @@ src/bin # Ignore the example folder (personal to each user) examples/* +# Ignore the example folder (personal to each user) +results/* -# Ignore System Files +# Ignore System Files (from MAC) .DS_Store ._.DS_Store # ignore the venv src/analyzer/venv - -src/analyzer/dashboard/static/plots diff --git a/src/analyzer/armiarma-analyzer.py b/src/analyzer/armiarma-analyzer.py index adb98af..538a87f 100644 --- a/src/analyzer/armiarma-analyzer.py +++ b/src/analyzer/armiarma-analyzer.py @@ -752,7 +752,7 @@ def main(): ## -- website code -- print("\n") - print("Results from crawler run on [month] running for [crawling time].
Total amount of peers on the peerstore:", peerstoreLen,".
Number of clients with the TPC port at 13000 (Prysm?):", cnt13000,".
Percentage of 'Prysm' peers from the peerstore (based on the TCP port):", round((cnt13000*100)/peerstoreLen,2),"%.
We manage to connect with", succeed,"peers from the peerstore.
This would be the distribution.") + print("Results from crawler run on [month] running for [crawling time].\n
Total amount of peers on the peerstore:", peerstoreLen,".\n
Number of clients with the TPC port at 13000 (Prysm?):", cnt13000,".\n
Percentage of 'Prysm' peers from the peerstore (based on the TCP port):", round((cnt13000*100)/peerstoreLen,2),"%.\n
We manage to connect with", succeed,"peers from the peerstore.\n
This would be the distribution.") print("\n") diff --git a/src/analyzer/total-overview-analysis.py b/src/analyzer/total-overview-analysis.py index 5e30ea1..35857ab 100644 --- a/src/analyzer/total-overview-analysis.py +++ b/src/analyzer/total-overview-analysis.py @@ -6,28 +6,41 @@ import json import pandas as pd import matplotlib.pyplot as plt +import numpy as np def mainexecution(): projectsFolder = sys.argv[1] + outFolder = sys.argv[2] + outJson = outFolder + '/' + 'client-distribution.json' # So far, the generated panda will just contain the client count p = {'date': [], 'Lighthouse': [], 'Teku': [], 'Nimbus': [], 'Prysm': [], 'Lodestar': [], 'Unknown': []} + # Concatenation of the Json values + j = [] print(projectsFolder) for root, dirs, _ in os.walk(projectsFolder): print(dirs) + dirs.sort() # We just need to access the folders inside examples for d in dirs: fold = os.path.join(root, d) f = fold + '/metrics/custom-metrics.json' if os.path.exists(f): # Load the readed values from the json into the panda - poblatePandaCustomMetrics(p, f) + poblatePandaCustomMetrics(p, j, f) break + + # Export the Concatenated json to the given folder + with open(outJson, 'w', encoding='utf-8') as f: + json.dump(j, f, ensure_ascii=False, indent=4) + # After filling the dict with all the info from the JSONs, generate the panda df = pd.DataFrame (p, columns = ['date', 'Lighthouse', 'Teku', 'Nimbus', 'Prysm', 'Lodestar', 'Unknown']) + df = df.sort_values(by="date") print(df) + outputFigsFolder = outFolder + '/' + 'plots' clientList = ['Lighthouse', 'Teku', 'Nimbus', 'Prysm', 'Lodestar', 'Unknown'] figSize = (10,6) titleSize = 24 @@ -36,12 +49,16 @@ def mainexecution(): # Plot the images plotStackedChart(df, opts={ 'figSize': figSize, - 'figTitle': 'CrawlSummary.png', + 'figTitle': 'Client-Distribution.png', + 'figtitle': 'client-distribution.png', + 'outputPath': outputFigsFolder, 'align': 'center', + 'grid': 'y', 'textSize': textSize, - 'title': "Evolution of the observed client distribution", + 'title': "Evolution of the projected client distribution", 'ylabel': 'Client concentration (%)', - 'xlabel': None, + 'xlabel': None, + 'yticks': np.arange(0, 110, 10), 'legendLabels': clientList, 'titleSize': titleSize, 'labelSize': labelSize, @@ -50,24 +67,43 @@ def mainexecution(): 'tickRotation': 90, 'show': True}) -def poblatePandaCustomMetrics(p, jsonFile): - print('poblating panda') +def poblatePandaCustomMetrics(p, j, jsonFile): # Read the Json jsf = open(jsonFile) jsonValues = json.load(jsf) jsf.close() + + # Add readed Json to the previous ones (concatenate them) + j.append(jsonValues) # Compose the date of the crawling day cday = str(jsonValues['StartTime']['Year']) + '-' + str(jsonValues['StartTime']['Month']) + '-' + str(jsonValues['StartTime']['Day']) p['date'].append(cday) - ps = jsonValues['PeerStore']['ConnectionSucceed']['Total'] + ps = jsonValues['PeerStore']['Total'] # Get percentages for each of the clients - lig = (100 * int(jsonValues['PeerStore']['ConnectionSucceed']['Lighthouse']['Total'])) / ps - tek = (100 * int(jsonValues['PeerStore']['ConnectionSucceed']['Teku']['Total'])) / ps - nim = (100 * int(jsonValues['PeerStore']['ConnectionSucceed']['Nimbus']['Total'])) / ps - pry = (100 * int(jsonValues['PeerStore']['ConnectionSucceed']['Prysm']['Total'])) / ps - lod = (100 * int(jsonValues['PeerStore']['ConnectionSucceed']['Lodestar']['Total'])) / ps - unk = (100 * int(jsonValues['PeerStore']['ConnectionSucceed']['Unknown']['Total'])) / ps + crwlig = jsonValues['PeerStore']['ConnectionSucceed']['Lighthouse']['Total'] + crwtek = jsonValues['PeerStore']['ConnectionSucceed']['Teku']['Total'] + crwnim = jsonValues['PeerStore']['ConnectionSucceed']['Nimbus']['Total'] + crwlod = jsonValues['PeerStore']['ConnectionSucceed']['Lodestar']['Total'] + crwunk = jsonValues['PeerStore']['ConnectionSucceed']['Unknown']['Total'] + + prysPort = jsonValues['PeerStore']['Port13000'] + res = ps - prysPort + + noPrysm = crwlig + crwtek + crwnim + crwlod + crwunk + + estimlig = (res*crwlig)/noPrysm + estimtek = (res*crwtek)/noPrysm + estimnim = (res*crwnim)/noPrysm + estimlod = (res*crwlod)/noPrysm + estimunk = (res*crwunk)/noPrysm + + lig = round((estimlig*100)/ps, 2) + tek = round((estimtek*100)/ps, 2) + nim = round((estimnim*100)/ps, 2) + pry = round((prysPort*100)/ps, 2) + lod = round((estimlod*100)/ps, 2) + unk = round((estimunk*100)/ps, 2) p['Lighthouse'].append(lig) p['Teku'].append(tek) @@ -78,7 +114,8 @@ def poblatePandaCustomMetrics(p, jsonFile): def plotStackedChart(p, opts): - + outputFile = str(opts['outputPath']) + '/' + opts['figTitle'] + ax = p.plot.area(figsize = opts['figSize'], x='date', stacked=True) # labels @@ -90,9 +127,14 @@ def plotStackedChart(p, opts): handles,legends = ax.get_legend_handles_labels() ax.legend(handles=handles, labels=opts['legendLabels'], loc='upper center', bbox_to_anchor=(0.5, -0.05), fancybox=True, shadow=True, ncol=6) + if opts['grid'] != None: + ax.grid(which='major', axis=opts['grid'], linestyle='--') + + plt.yticks(opts['yticks']) + plt.title(opts['title'], fontsize = opts['titleSize']) plt.tight_layout() - #plt.savefig(outputFile) + plt.savefig(outputFile) if opts['show'] is True: plt.show() From 653403cdb1a654e2bfcb615f81daa88ffbe708d6 Mon Sep 17 00:00:00 2001 From: Mikel Cortes Date: Tue, 25 May 2021 13:06:19 +0200 Subject: [PATCH 4/5] Include the -o flag for a projects overview --- armiarma.sh | 68 ++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 62 insertions(+), 6 deletions(-) diff --git a/armiarma.sh b/armiarma.sh index 2e54053..a5af660 100755 --- a/armiarma.sh +++ b/armiarma.sh @@ -30,6 +30,9 @@ Help() echo " *Parameters for -p [name]" echo " -f Run a time specified test, performing the analysis" echo " of the obtained" + echo " *Parameters for -f [network] [project-name] [time](minutes)" + echo " -o Run the general analysis over the entire projects' folder." + echo "" echo " Parameters:" echo " [network] The ETH2 network where the crawler will be running" echo " Currently supported networks:" @@ -37,6 +40,9 @@ Help() echo " [project-name] Specify the name of the folder where the metrics" echo " and plots will be stored." echo " Find them on 'armiarma/examples/[project-name]'" + echo " [time] Specific time in minutes to run the crawler, performing afterwards" + echo " the general analysis." + echo "" echo "" echo " BSC-ETH2 TEAM" } @@ -142,7 +148,7 @@ LaunchCrawler(){ # Move to the example folder cd "./examples/${folderName}" - # Append the bash env variables to the temp file + # Append the bash env variables to the temp fhiustile echo "metricsFolder=\"${metricsFolder}\"" >> config.sh echo "armiarmaPath=\"${folderPath}\"" >> config.sh @@ -211,7 +217,7 @@ LaunchAnalyzer(){ echo "venv already created" else echo "Generating the virtual env" - python -m virtualenv "$VENV" + python3 -m virtualenv "$VENV" fi echo "" # Source the virtual env @@ -230,7 +236,7 @@ LaunchAnalyzer(){ # -- END TEMP -- echo "Checking if Python dependencies are installed..." - pip install -r ./src/analyzer/requirements.txt + pip3 install -r ./src/analyzer/requirements.txt echo "" # Set the Paths for the gossip-metrics.json peerstore.json and output @@ -249,13 +255,45 @@ LaunchAnalyzer(){ # Run the Analyzer echo " Launching analyzer" echo "" - python ./src/analyzer/armiarma-analyzer.py "$csv" "$peerstore" "$extrametrics" "$plots" + python3 ./src/analyzer/armiarma-analyzer.py "$csv" "$peerstore" "$extrametrics" "$plots" # Deactivate the VENV deactivate } +# Launch the General Overview of all the projects in the examples folder +LaunchGeneralResults(){ + if [[ -d "./general-results/plots" ]]; then + echo "" + else + mkdir "./general-results/plots" + fi + + # Source the virtual env + source "${VENV}/bin/activate" + + # Check if the virtual env is created + venvPath="${PWD}/src/analyzer/venv" + if [[ "$VIRTUAL_ENV" = "$venvPath" ]] + then + echo "VENV successfuly sourced" + else + echo "ERROR. VENV was unable to source" >&2 + fi + echo "" + + # Run the Analyzer + echo " Launching General Overview Analyzer" + echo "" + python3 ./src/analyzer/total-overview-analysis.py ./examples ./results + echo "results available in \$ARMIARMA/results" + echo "" + # Deactivate the VENV + deactivate +} + + # -------- END OF FUNCTION DEFINITION --------- # Execution Secuence @@ -263,6 +301,7 @@ LaunchAnalyzer(){ # 0. Get the options go version +# Generate the examples folder if [[ -d ./examples ]]; then echo "" echo " ----------- ARMIARMA $version -----------" @@ -274,6 +313,15 @@ else echo "" mkdir ./examples fi +# Generate the general-results folder +if [[ -d ./general-results ]]; then + echo "" +else + echo "" + echo "Generating ./general-results folder" + echo "" + mkdir ./general-results +fi # Check if any argument was given. # If not, print Help and exit @@ -283,7 +331,7 @@ if [[ -z "$1" ]]; then exit 1 fi -while getopts ":hcpfdts" option; do +while getopts ":hcpfdo" option; do case $option in h) # display Help Help @@ -329,10 +377,18 @@ while getopts ":hcpfdts" option; do echo "Calling the Analyzer" LaunchAnalyzer "$folderName" "$folderPath" - + echo "Exit Crawler execution" exit;; + o) # Generate the general overview of the previously generated projects + + LaunchGeneralResults + + echo "Overview Analyzer Finished!" + echo "" + exit;; + d) # Option to run Armiarma Crawler for a specific given time on the iexec decentralice platform # processing the results with the Analyzer and returning the metrics and plots in a zip file # TODO: encrypt the results with a certain security measure, so noone can access to them From cb32c7264545afcd5abf1a2b5fdd991583fcf5ff Mon Sep 17 00:00:00 2001 From: Mikel Cortes Date: Tue, 25 May 2021 13:29:31 +0200 Subject: [PATCH 5/5] Update Readme with Tool updates --- .gitignore | 2 +- README.md | 6 +++++- armiarma.sh | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 651d89c..55ab6ab 100644 --- a/.gitignore +++ b/.gitignore @@ -9,7 +9,7 @@ src/bin # Ignore the example folder (personal to each user) examples/* # Ignore the example folder (personal to each user) -results/* +general-results/* # Ignore System Files (from MAC) .DS_Store diff --git a/README.md b/README.md index fca272a..319f736 100644 --- a/README.md +++ b/README.md @@ -9,17 +9,21 @@ To use the tool the following tools needs to be installed on the machine: ### Usage The crawler can be easily executed from the `armiarma.sh` file (make sure that is an executable file). -The executable `armiarma.sh` launching shell script supports three different commands that correspond to the main functionalities of the tool. +The executable `armiarma.sh` launching shell script supports five different commands that correspond to the main functionalities of the tool. Those commands are: - `./armiarma.sh -h` to display the help text of the tool. - `./armiarma.sh -c [network] [project-name]` to launch the armiarma crawler on the given network. The given project-name is the name of the folder where the gathered metrics and where all the related information will be saved. The folder will be placed on the `./examples` folder. - `./armiarma.sh -p [project-name]` to launch the armiarma analyzer over the metrics from the given project. The generted plots of the analysis will be saved on the `./examples/[project-name]/plots` folder. +- `./armiarma.sh -f [network] [project-name] [time]` to launch the armiarma crawler on the given network for the given `time` in minutes. The given project-name is the name of the folder where the gathered metrics and where all the related information will be saved. The folder will be placed on the `./examples` folder. After the crawling for the given time, the analyzer will be launched automatically. The summary results will be saved on the `./examples/[project-name]/plots` folder. +- `./armiarma.sh -o` to launch the armiarma general overview analyzer over the metrics from the projects' folder. The generted results of the analysis will be saved on the `./general-resutls` folder. ``` ./armiarma.sh --> -h '-> -c [network] [project-name] '-> -p [project-name] + '-> -f [network] [project-name] [time](minutes) + '-> -o ``` The tool will get the Go packages and compile Rumor as generate the virtual env and install the python3 dependencies for the user. diff --git a/armiarma.sh b/armiarma.sh index a5af660..16e3e31 100755 --- a/armiarma.sh +++ b/armiarma.sh @@ -286,7 +286,7 @@ LaunchGeneralResults(){ # Run the Analyzer echo " Launching General Overview Analyzer" echo "" - python3 ./src/analyzer/total-overview-analysis.py ./examples ./results + python3 ./src/analyzer/total-overview-analysis.py ./examples ./general-results echo "results available in \$ARMIARMA/results" echo "" # Deactivate the VENV