Improvements (#67)

* TOOLS-1224: (ASADM-HEALTHCHECK) Add new rules to check server health stats.

* TOOLS-1323: (ASADM) Show the 'show config' output for specific namespaces matching pattern listed after ‘for’.

* TOOLS-1324: (ASADM) Add all the peers and alumni list details to collectinfo.

* TOOLS-1325: (ASADM) Fix to not show online access addresses in offline alumni node list.

* TOOLS-1334: (TOOLS) Remove SSLv3 support.

lib/basiccontroller.py

@@ -1,4 +1,4 @@
-# Copyright 2013-2018 Aerospike, Inc.
+# Copyright 2013-2019 Aerospike, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -482,7 +482,7 @@ def _do_default(self, line):
 class ShowConfigController(BasicCommandController):
 
     def __init__(self):
-        self.modifiers = set(['with', 'like', 'diff'])
+        self.modifiers = set(['with', 'like', 'diff', 'for'])
         self.getter = GetConfigController(self.cluster)
 
     @CommandHelp('Displays service, network, and namespace configuration',
@@ -546,7 +546,7 @@ def do_namespace(self, line):
                 arg="-flip", default=False, modifiers=self.modifiers,
                 mods=self.mods)
 
-        ns_configs = self.getter.get_namespace(nodes=self.nodes)
+        ns_configs = self.getter.get_namespace(nodes=self.nodes, for_mods=self.mods['for'])
 
         return [util.Future(self.view.show_config,
             "%s Namespace Configuration" % (ns), configs, self.cluster,
@@ -1038,6 +1038,7 @@ def _get_as_metadata(self):
         endpoints = util.Future(self.cluster.info_service_list, nodes=self.nodes).start()
         services = util.Future(self.cluster.info_peers_flat_list, nodes=self.nodes).start()
         udf_data = util.Future(self.cluster.info_udf_list, nodes=self.nodes).start()
+        health_outliers = util.Future(self.cluster.info_health_outliers, nodes=self.nodes).start()
 
         builds = builds.result()
         editions = editions.result()
@@ -1047,6 +1048,7 @@ def _get_as_metadata(self):
         endpoints = endpoints.result()
         services = services.result()
         udf_data = udf_data.result()
+        health_outliers = health_outliers.result()
 
         for nodeid in builds:
             metamap[nodeid] = {}
@@ -1058,6 +1060,7 @@ def _get_as_metadata(self):
             self._get_meta_for_sec(endpoints, 'endpoints', nodeid, metamap)
             self._get_meta_for_sec(services, 'services', nodeid, metamap)
             self._get_meta_for_sec(udf_data, 'udf', nodeid, metamap)
+            self._get_meta_for_sec(health_outliers, 'health', nodeid, metamap)
 
         return metamap
 
@@ -1200,12 +1203,17 @@ def _dump_collectinfo_pretty_print(self, timestamp, as_logfile_prefix, config_pa
 
         collect_output = time.strftime("%Y-%m-%d %H:%M:%S UTC\n", timestamp)
 
-        dignostic_info_params = [
-            'network', 'namespace', 'set', 'xdr', 'dc', 'sindex']
+        dignostic_info_params = ['network', 'namespace', 'set', 'xdr', 'dc', 'sindex']
+
         dignostic_features_params = ['features']
-        dignostic_show_params = ['config', 'config xdr', 'config dc', 'config cluster', 'distribution', 'distribution eviction',
-                                 'distribution object_size -b', 'latency', 'statistics', 'statistics xdr', 'statistics dc', 'statistics sindex', 'pmap']
-        dignostic_aerospike_cluster_params = ['service', 'services', 'roster:']
+
+        dignostic_show_params = ['config', 'config xdr', 'config dc', 'config cluster', 'distribution',
+                                 'distribution eviction', 'distribution object_size -b', 'latency',
+                                 'statistics', 'statistics xdr', 'statistics dc', 'statistics sindex', 'pmap']
+
+        dignostic_aerospike_cluster_params = ['service', 'services', 'peers-clear-std', 'peers-clear-alt',
+                                              'peers-tls-std', 'peers-tls-alt', 'alumni-clear-std',
+                                              'alumni-tls-std', 'peers-generation', 'roster:']
 
         summary_params = ['summary']
         summary_info_params = ['network', 'namespace', 'set', 'xdr', 'dc', 'sindex']
@@ -1555,6 +1563,8 @@ def _get_as_meta_data(self, stanza):
                 editions_in_shortform[node] = util.convert_edition_to_shortform(edition)
 
             return editions_in_shortform
+        elif stanza == "health":
+            return self.cluster.info_health_outliers(nodes=self.nodes)
 
     @CommandHelp(
         'Displays health summary. If remote server System credentials provided, then it will collect remote system stats',
@@ -1733,6 +1743,10 @@ def _do_default(self, line):
                     ("udf", "UDF",
                      [("CLUSTER", cluster_name), ("NODE", None), (None, None), ("FILENAME", None)]),
                 ]),
+                "health": (self._get_as_meta_data, [
+                    ("health", "METADATA",
+                     [("CLUSTER", cluster_name), ("NODE", None), (None, None), ("OUTLIER", None)]),
+                ]),
             }
             sys_cmd_dict = {
                 "sys_stats": (util.restructure_sys_data, [

lib/client/cluster.py

@@ -167,8 +167,9 @@ def get_down_nodes(self):
                     # in case of using alumni services, we might have offline
                     # nodes which can't detect online nodes
                     continue
+
                 alumni_peers = util.flatten(node.get_alumni_peers())
-                peers = util.flatten(node.get_peers())
+                peers = util.flatten(node.get_peers(all=True))
                 not_visible = set(alumni_peers) - set(peers)
                 if len(not_visible) >= 1:
                     for n in not_visible:

lib/client/node.py

@@ -663,7 +663,7 @@ def get_alumni_peers(self):
             # Unlike services-alumni, info_peers_alumni for server version prior to 4.3.1 gives
             # only old nodes (which are not part of current cluster), so to get full list we need to
             # add info_peers
-            alumni_peers = self.info_peers()
+            alumni_peers = self.get_peers()
             return list(set(alumni_peers + self.info_peers_alumni()))
         else:
             alumni_services = self.info_services_alumni()
@@ -672,14 +672,20 @@ def get_alumni_peers(self):
             return self.info_services()
 
     @return_exceptions
-    def get_peers(self):
+    def get_peers(self, all=False):
         if self.use_peers_list:
+            if all:
+                return self.info_peers_alt() + self.info_peers()
+
             if self.use_services_alt:
                 return self.info_peers_alt()
 
             return self.info_peers()
 
         else:
+            if all:
+                return self.info_services_alt() + self.info_services()
+
             if self.use_services_alt:
                 return self.info_services_alt()
 
@@ -848,6 +854,22 @@ def info_set_statistics(self):
             set_dict.update(stat)
 
         return sets
+
+    @return_exceptions
+    def info_health_outliers(self):
+        stats = self.info("health-outliers")
+        stats = util.info_to_list(stats)
+        if not stats:
+            return {}
+        stats = [util.info_colon_to_dict(stat) for stat in stats]
+        health_dict = {}
+
+        for i, stat  in enumerate(stats):
+            key = "outlier" + str(i)
+            health_dict[key] = stat
+
+        return health_dict
+
 
     @return_exceptions
     def info_bin_statistics(self):

lib/client/ssl_context.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 
-# Copyright 2013-2018 Aerospike, Inc.
+# Copyright 2013-2019 Aerospike, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -369,10 +369,10 @@ def _verify_cb(self, conn, cert, errnum, depth, ok):
         return ok
 
     def _parse_protocols(self, protocols):
-        protocols_to_disable = [
-            "SSLv2", "SSLv3", "TLSv1", "TLSv1.1", "TLSv1.2"]
+        all_protocols = ["TLSv1", "TLSv1.1", "TLSv1.2"]
         protocols_to_enable = set()
         method = None
+
         if not protocols:
             try:
                 method = SSL.TLSv1_2_METHOD
@@ -384,76 +384,88 @@ def _parse_protocols(self, protocols):
             protocols = protocols.split()
             for proto in protocols:
                 try:
-                    if proto == "SSLv3" or proto == "+SSLv3":
-                        protocols_to_enable.add("SSLv3")
-                    elif proto == "-SSLv3":
-                        protocols_to_enable.remove("SSLv3")
-                    elif proto == "TLSv1" or proto == "+TLSv1":
+                    if proto == "TLSv1" or proto == "+TLSv1":
                         protocols_to_enable.add("TLSv1")
                     elif proto == "-TLSv1":
                         protocols_to_enable.remove("TLSv1")
+
                     elif proto == "TLSv1.1" or proto == "+TLSv1.1":
                         protocols_to_enable.add("TLSv1.1")
                     elif proto == "-TLSv1.1":
                         protocols_to_enable.remove("TLSv1.1")
+
                     elif proto == "TLSv1.2" or proto == "+TLSv1.2":
                         protocols_to_enable.add("TLSv1.2")
                     elif proto == "-TLSv1.2":
                         protocols_to_enable.remove("TLSv1.2")
+
                     elif proto == "all" or proto == "+all":
-                        protocols_to_enable.add("SSLv3")
-                        protocols_to_enable.add("TLSv1")
-                        protocols_to_enable.add("TLSv1.1")
-                        protocols_to_enable.add("TLSv1.2")
+                        protocols_to_enable += all_protocols
                     elif proto == "-all":
                         protocols_to_enable.clear()
+
                     elif proto == "SSLv2" or proto == "+SSLv2":
-                        raise Exception(
-                            "Protocol SSLv2 not supported (RFC 6176)")
+                        raise Exception("Protocol SSLv2 not supported (RFC 6176)")
                     elif proto == "-SSLv2":
                         continue
+
+                    elif proto == "SSLv3" or proto == "+SSLv3":
+                        raise Exception("Protocol SSLv3 not supported")
+                    elif proto == "-SSLv3":
+                        continue
+
                     else:
-                        raise Exception(
-                            "Wrong protocol entry %s" % (str(proto)))
+                        raise Exception("Wrong protocol entry %s" % (str(proto)))
+
                 except KeyError:
                     pass
+
         if not protocols_to_enable:
             raise Exception("Wrong protocol entries")
+
         protocols_to_enable = list(protocols_to_enable)
+
         if len(protocols_to_enable) > 1:
             # Multiple protocols are enabled
             method = SSL.SSLv23_METHOD
         else:
-            if protocols_to_enable[0] == "SSLv3":
-                method = SSL.SSLv3_METHOD
-            elif protocols_to_enable[0] == "TLSv1":
+            if protocols_to_enable[0] == "TLSv1":
                 method = SSL.TLSv1_METHOD
+
             elif protocols_to_enable[0] == "TLSv1.1":
                 try:
                     method = SSL.TLSv1_1_METHOD
                 except Exception:
                     raise Exception(
-                        "No support to protocol %s. Wrong OpenSSL or Python version. Please use PyOpenSSL >= 0.15." % ("TLSv1.1"))
+                        "No support to protocol TLSv1.1. Wrong OpenSSL or Python version. Please use PyOpenSSL >= 0.15.")
+
             elif protocols_to_enable[0] == "TLSv1.2":
                 try:
                     method = SSL.TLSv1_2_METHOD
                 except Exception:
                     raise Exception(
-                        "No support to protocol %s. Wrong OpenSSL or Python version. Please use PyOpenSSL >= 0.15." % ("TLSv1.2"))
-        protocols_to_disable = list(
-            set(protocols_to_disable) - set(protocols_to_enable))
+                        "No support to protocol TLSv1.2. Wrong OpenSSL or Python version. Please use PyOpenSSL >= 0.15.")
+
+        protocols_to_disable = list(set(all_protocols) - set(protocols_to_enable))
+
         return method, protocols_to_disable
 
     def _set_context_options(self, ctx, protocols_to_disable):
+        try:
+            # always disable SSLv2, as per RFC 6176
+            ctx.set_options(SSL.OP_NO_SSLv2)
+
+            # aerospike does not support SSLv3
+            ctx.set_options(SSL.OP_NO_SSLv3)
+        except Exception:
+            pass
+
         if not protocols_to_disable:
             return ctx
+
         for proto in protocols_to_disable:
             try:
-                if proto == "SSLv2":
-                    ctx.set_options(SSL.OP_NO_SSLv2)
-                elif proto == "SSLv3":
-                    ctx.set_options(SSL.OP_NO_SSLv3)
-                elif proto == "TLSv1":
+                if proto == "TLSv1":
                     ctx.set_options(SSL.OP_NO_TLSv1)
                 elif proto == "TLSv1.1":
                     ctx.set_options(SSL.OP_NO_TLSv1_1)

lib/collectinfocontroller.py

@@ -966,6 +966,10 @@ def _do_default(self, line):
                     ("udf", "UDF", "METADATA", True, [
                      ("CLUSTER", cluster_name), ("NODE", None), (None, None), ("FILENAME", None)]),
                 ]),
+                "health": (self.loghdlr.info_meta_data, [
+                    ("health", "METADATA", "HEALTH", True, [
+                     ("CLUSTER", cluster_name), ("NODE", None), (None, None), ("OUTLIER", None)]),
+                ]),
 
                 "sys_stats": (self.loghdlr.get_sys_data, [
                     ("free-m", "SYSTEM", "FREE", True,

lib/getcontroller.py

@@ -181,19 +181,30 @@ def get_network(self, flip=True, nodes='all'):
 
         return network_configs
 
-    def get_namespace(self, flip=True, nodes='all'):
-        configs = self.cluster.info_get_config(
-            nodes=nodes, stanza='namespace')
-        for node in configs:
-            if isinstance(configs[node], Exception):
-                configs[node] = {}
+    def get_namespace(self, flip=True, nodes='all', for_mods=[]):
+        namespaces = util.Future(self.cluster.info_namespaces, nodes=nodes).start().result()
+        namespaces = namespaces.values()
+        namespace_set = set()
 
-        ns_configs = {}
-        for node, node_config in configs.iteritems():
-            if not node_config or isinstance(node_config, Exception):
+        for namespace in namespaces:
+            if isinstance(namespace, Exception):
                 continue
 
-            ns_configs[node] = node_config
+            namespace_set.update(namespace)
+
+        namespace_list = util.filter_list(list(namespace_set), for_mods)
+        ns_configs = {}
+
+        for index, namespace in enumerate(namespace_list):
+            node_configs = util.Future(self.cluster.info_get_config, stanza='namespace', namespace=namespace, namespace_id=index, nodes=nodes).start().result()
+            for node, node_config in node_configs.items():
+                if not node_config or isinstance(node_config, Exception) or not namespace in node_config:
+                    continue
+
+                if node not in ns_configs:
+                    ns_configs[node] = {}
+
+                ns_configs[node][namespace] = node_config[namespace]
 
         if flip:
             ns_configs = util.flip_keys(ns_configs)

lib/health/parser.py

@@ -54,6 +54,7 @@ class HealthLexer(object):
         'ENVIRONMENT': 'ENVIRONMENT',
         'FREE': 'FREE',
         'HDPARM': 'HDPARM',
+        'HEALTH': 'HEALTH',
         'INTERRUPTS': 'INTERRUPTS',
         'IOSTAT': 'IOSTAT',
         'IPTABLES': 'IPTABLES',
@@ -97,6 +98,7 @@ class HealthLexer(object):
         'INTERRUPT_TYPE': 'INTERRUPT_TYPE',
         'KEY': 'KEY',
         'NODE': 'NODE',
+        'OUTLIER': 'OUTLIER',
         'SNAPSHOT': 'SNAPSHOT'
     }
 

lib/health/query.py

@@ -1737,4 +1737,17 @@
 
 SET CONSTRAINT VERSION ALL;
 
+/*
+Server Health Check
+*/
+
+SET CONSTRAINT VERSION >= 4.3.1;
+
+m = select * from METADATA.HEALTH save;
+ASSERT(m, False, "Outlier[s] detected by the server health check.", "OPERATIONS", WARNING,
+			    "Listed outlier[s] have been reported by the server health check and they might be misbehaving.",
+			    "Server health check outlier detection. Run command 'asinfo -v health-outliers' to see list of outliers");
+
+SET CONSTRAINT VERSION ALL;
+
 '''

lib/health/query/health.hql

@@ -1721,3 +1721,16 @@ ASSERT(r, False, "Roster is null or NOT set.", "OPERATIONS", CRITICAL,
 				"Roster null check.");
 
 SET CONSTRAINT VERSION ALL;
+
+/*
+Server Health Check
+*/
+
+SET CONSTRAINT VERSION >= 4.3.1;
+
+m = select * from METADATA.HEALTH save;
+ASSERT(m, False, "Outlier[s] detected by the server health check.", "OPERATIONS", WARNING,
+			    "Listed outlier[s] have been reported by the server health check and they might be misbehaving.",
+			    "Server health check outlier detection. Run command 'asinfo -v health-outliers' to see list of outliers");
+
+SET CONSTRAINT VERSION ALL;

version.txt

@@ -1 +1 @@
-0.2.1
+0.2.2