Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[explorer/nodewatch] task: add new route #343

Open
wants to merge 13 commits into
base: dev
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 9 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
86 changes: 57 additions & 29 deletions explorer/nodewatch/nodewatch/NetworkRepository.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,10 @@ def __init__(
height=0,
finalized_height=0,
balance=0,
is_healthy=None,
is_https_enabled=None,
is_wss_enabled=None,
rest_version=None,
roles=0xFF):
"""Creates a descriptor."""

Expand All @@ -37,6 +41,10 @@ def __init__(
self.height = height
self.finalized_height = finalized_height
self.balance = balance
self.is_healthy = is_healthy
self.is_https_enabled = is_https_enabled
self.is_wss_enabled = is_wss_enabled
self.rest_version = rest_version
self.roles = roles

@property
Expand All @@ -57,6 +65,10 @@ def to_json(self):
'height': self.height,
'finalizedHeight': self.finalized_height,
'balance': self.balance,
'isHealthy': self.is_healthy,
'isHttpsEnabled': self.is_https_enabled,
'isWssEnabled': self.is_wss_enabled,
'restVersion': self.rest_version,
'roles': self.roles
}

Expand Down Expand Up @@ -150,37 +162,31 @@ def load_node_descriptors(self, nodes_data_filepath):
# sort by name
self.node_descriptors.sort(key=lambda descriptor: descriptor.name)

def _create_descriptor_from_json(self, json_node):
# network crawler extracts as much extra data as possible, but it might not always be available for all nodes
extra_data = (0, 0, 0)
if 'extraData' in json_node:
json_extra_data = json_node['extraData']
extra_data = (json_extra_data.get('height', 0), json_extra_data.get('finalizedHeight', 0), json_extra_data.get('balance', 0))
def _handle_nem_node(self, json_node, extra_data):
network_identifier = json_node['metaData']['networkId']
if network_identifier < 0:
network_identifier += 0x100

if self.is_nem:
network_identifier = json_node['metaData']['networkId']
if network_identifier < 0:
network_identifier += 0x100

if self._network.identifier != network_identifier:
return None

node_protocol = json_node['endpoint']['protocol']
node_host = json_node['endpoint']['host']
node_port = json_node['endpoint']['port']

json_identity = json_node['identity']
main_public_key = PublicKey(json_identity['public-key'])
node_public_key = PublicKey(json_identity['node-public-key']) if 'node-public-key' in json_identity else None
return NodeDescriptor(
self._network.public_key_to_address(main_public_key),
main_public_key,
node_public_key,
f'{node_protocol}://{node_host}:{node_port}',
json_identity['name'],
json_node['metaData']['version'],
*extra_data)
if self._network.identifier != network_identifier:
return None

node_protocol = json_node['endpoint']['protocol']
node_host = json_node['endpoint']['host']
node_port = json_node['endpoint']['port']

json_identity = json_node['identity']
main_public_key = PublicKey(json_identity['public-key'])
node_public_key = PublicKey(json_identity['node-public-key']) if 'node-public-key' in json_identity else None
return NodeDescriptor(
self._network.public_key_to_address(main_public_key),
main_public_key,
node_public_key,
f'{node_protocol}://{node_host}:{node_port}',
json_identity['name'],
json_node['metaData']['version'],
*extra_data)

def _handle_symbol_node(self, json_node, extra_data):
symbol_endpoint = ''
roles = json_node['roles']
has_api = bool(roles & 2)
Expand All @@ -192,6 +198,15 @@ def _create_descriptor_from_json(self, json_node):
if self._network.generation_hash_seed != Hash256(json_node['networkGenerationHashSeed']):
return None

api_node_info_data = (None, None, None, None)
if 'apiNodeInfo' in json_node:
Jaguar0625 marked this conversation as resolved.
Show resolved Hide resolved
json_api_node_info_data = json_node['apiNodeInfo']
api_node_info_data = (
json_api_node_info_data.get('isHealth', None),
json_api_node_info_data.get('isHttpsEnabled', None),
json_api_node_info_data.get('isWssEnabled', None),
json_api_node_info_data.get('restVersion', None))

main_public_key = PublicKey(json_node['publicKey'])
node_public_key = PublicKey(json_node['nodePublicKey']) if 'nodePublicKey' in json_node else None
return NodeDescriptor(
Expand All @@ -202,8 +217,21 @@ def _create_descriptor_from_json(self, json_node):
json_node['friendlyName'],
self._format_symbol_version(json_node['version']),
*extra_data,
*api_node_info_data,
roles)

def _create_descriptor_from_json(self, json_node):
# network crawler extracts as much extra data as possible, but it might not always be available for all nodes
extra_data = (0, 0, 0)
if 'extraData' in json_node:
json_extra_data = json_node['extraData']
extra_data = (json_extra_data.get('height', 0), json_extra_data.get('finalizedHeight', 0), json_extra_data.get('balance', 0))

if self.is_nem:
return self._handle_nem_node(json_node, extra_data)

return self._handle_symbol_node(json_node, extra_data)

@staticmethod
def _format_symbol_version(version):
version_parts = [(version >> 24) & 0xFF, (version >> 16) & 0xFF, (version >> 8) & 0xFF, version & 0xFF]
Expand Down
43 changes: 37 additions & 6 deletions explorer/nodewatch/nodewatch/RoutesFacade.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import asyncio
import datetime
import random

from zenlog import log

Expand Down Expand Up @@ -65,15 +66,45 @@ def html_nodes(self):
'explorer_endpoint': self.explorer_endpoint
})

def json_nodes(self, role, exact_match=False):
"""Returns all nodes with matching role."""
def json_nodes(self, **kwargs):
"""Returns all nodes with condition."""

def role_filter(descriptor):
return role == descriptor.roles if exact_match else role == (role & descriptor.roles)
role = kwargs.get('role', None)
exact_match = kwargs.get('exact_match', False)
limit = kwargs.get('limit', None)
ssl = kwargs.get('ssl', None)
order = kwargs.get('order', None)

return list(map(
def custom_filter(descriptor):
role_condition = True

if role is not None:
role_condition = role == descriptor.roles if exact_match else role == (role & descriptor.roles)

if ssl is not None:
ssl_condition = (descriptor.is_https_enabled and descriptor.is_wss_enabled)
return role_condition and ssl_condition

return role_condition

nodes = list(map(
lambda descriptor: descriptor.to_json(),
filter(role_filter, self.repository.node_descriptors)))
filter(custom_filter, self.repository.node_descriptors)))

if order == 'random':
random.shuffle(nodes)

if limit is not None:
nodes = nodes[:limit]

return nodes

def json_node(self, filter_field, public_key):
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

you might want to consider checking if public_key is proper public key and returning 400 if not

"""Returns a node with matching public key."""

matching_items = [item.to_json() for item in self.repository.node_descriptors if item.to_json()[filter_field] == public_key]
Jaguar0625 marked this conversation as resolved.
Show resolved Hide resolved

return next(iter(matching_items), None)

def json_height_chart(self):
"""Builds a JSON height chart."""
Expand Down
43 changes: 40 additions & 3 deletions explorer/nodewatch/nodewatch/__init__.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
from enum import Enum
from pathlib import Path

from apscheduler.schedulers.background import BackgroundScheduler
Expand All @@ -11,6 +12,17 @@
from .RoutesFacade import MIN_HEIGHT_CLUSTER_SIZE, TIMESTAMP_FORMAT, NemRoutesFacade, SymbolRoutesFacade


class Field(Enum):
MAIN_PUBLIC_KEY = 'mainPublicKey'
NODE_PUBLIC_KEY = 'nodePublicKey'


def str_to_bool(value):
Jaguar0625 marked this conversation as resolved.
Show resolved Hide resolved
if value.lower() == 'true':
return True
return None


def create_app():
# pylint: disable=too-many-locals

Expand Down Expand Up @@ -61,7 +73,7 @@ def nem_summary(): # pylint: disable=unused-variable

@app.route('/api/nem/nodes')
def api_nem_nodes(): # pylint: disable=unused-variable
return jsonify(nem_routes_facade.json_nodes(1))
return jsonify(nem_routes_facade.json_nodes(role=1))

@app.route('/api/nem/chart/height')
def api_nem_chart_height(): # pylint: disable=unused-variable
Expand Down Expand Up @@ -91,13 +103,38 @@ def symbol_summary(): # pylint: disable=unused-variable
template_name, context = symbol_routes_facade.html_summary()
return render_template(template_name, **context)

def _get_json_nodes(role, exact_match, ssl, limit, order):
if ssl is not None:
ssl = str_to_bool(ssl)
Jaguar0625 marked this conversation as resolved.
Show resolved Hide resolved

if limit is not None:
limit = int(limit)

return jsonify(symbol_routes_facade.json_nodes(role=role, exact_match=exact_match, ssl=ssl, limit=limit, order=order))
Jaguar0625 marked this conversation as resolved.
Show resolved Hide resolved

@app.route('/api/symbol/nodes/api')
def api_symbol_nodes_api(): # pylint: disable=unused-variable
return jsonify(symbol_routes_facade.json_nodes(2, exact_match=True))
ssl = request.args.get('ssl', None)
limit = request.args.get('limit', None)
Jaguar0625 marked this conversation as resolved.
Show resolved Hide resolved
order = request.args.get('order', None)

return _get_json_nodes(2, True, ssl, limit, order)

@app.route('/api/symbol/nodes/peer')
def api_symbol_nodes_peer(): # pylint: disable=unused-variable
return jsonify(symbol_routes_facade.json_nodes(1))
ssl = request.args.get('ssl', None)
limit = request.args.get('limit', None)
order = request.args.get('order', None)
Jaguar0625 marked this conversation as resolved.
Show resolved Hide resolved

return _get_json_nodes(1, False, ssl, limit, order)

@app.route('/api/symbol/nodes/mainPublicKey/<main_public_key>')
def api_symbol_nodes_get_main_public_key(main_public_key): # pylint: disable=unused-variable
return jsonify(symbol_routes_facade.json_node(filter_field=Field.MAIN_PUBLIC_KEY.value, public_key=main_public_key))

@app.route('/api/symbol/nodes/nodePublicKey/<node_public_key>')
def api_symbol_nodes_get_node_public_key(node_public_key): # pylint: disable=unused-variable
return jsonify(symbol_routes_facade.json_node(filter_field=Field.NODE_PUBLIC_KEY.value, public_key=node_public_key))

@app.route('/api/symbol/chart/height')
def api_symbol_chart_height(): # pylint: disable=unused-variable
Expand Down
66 changes: 66 additions & 0 deletions explorer/nodewatch/tests/resources/symbol_nodes.json
Original file line number Diff line number Diff line change
Expand Up @@ -91,5 +91,71 @@
"host": "02.symbol-node.net",
"friendlyName": "Apple",
"nodePublicKey": "FBEAFCB15D2674ECB8DC1CD2C028C4AC0D463489069FDD415F30BB71EAE69864"
},
{
"apiNodeInfo": {
"isHealth": true,
"isHttpsEnabled": true,
"isWssEnabled": true,
"restVersion": "2.4.2"
},
"extraData": {
"balance": 101027.849383,
"finalizedHeight": 1486740,
"height": 1486762
},
"friendlyName": "xym pool",
"host": "xym.pool.me",
"networkGenerationHashSeed": "57F7DA205008026C776CB6AED843393F04CD458E0AA2D9F1D5F31A402072B2D6",
"networkIdentifier": 104,
"nodePublicKey": "FE7D3DBE8DDD219E1B20247DEBF150D9411EA5A312989103B037EFBD9D237DE0",
"port": 7900,
"publicKey": "A54CC798373F42B569AF21845CD0EBE755AB42EA04B3B8E2BE897166F89A971C",
"roles": 3,
"version": 16777987
},
{
"apiNodeInfo": {
"isHealth": true,
"isHttpsEnabled": false,
"isWssEnabled": false,
"restVersion": "2.4.2"
},
"extraData": {
"balance": 99.98108,
"finalizedHeight": 1486740,
"height": 1486760
},
"friendlyName": "yasmine farm",
"host": "yasmine.farm.tokyo",
"networkGenerationHashSeed": "57F7DA205008026C776CB6AED843393F04CD458E0AA2D9F1D5F31A402072B2D6",
"networkIdentifier": 104,
"nodePublicKey": "D05BE3101F2916AA34839DDC1199BE45092103A9B66172FA3D05911DC041AADA",
"port": 7900,
"publicKey": "5B20F8F228FF0E064DB0DE7951155F6F41EF449D0EC10960067C2BF2DCD61874",
"roles": 3,
"version": 16777988
},
{
"apiNodeInfo": {
"isHealth": true,
"isHttpsEnabled": true,
"isWssEnabled": true,
"restVersion": "2.4.2"
},
"extraData": {
"balance": 3155632.471994,
"finalizedHeight": 1486740,
"height": 1486762
},
"friendlyName": "Allnodes251",
"host": "",
"networkGenerationHashSeed": "57F7DA205008026C776CB6AED843393F04CD458E0AA2D9F1D5F31A402072B2D6",
"networkIdentifier": 104,
"nodePublicKey": "F25FDB3CD1A97DDC71A993D124E9BDD6518699F9F4016C29D341F53208D150D8",
"port": 7900,
"publicKey": "C69B5BDE17EEF7449C6D92856C1D295FE02AA3472651F042FB3FC2F771DAAF7B",
"roles": 2,
"version": 16777988
}
]
Loading