From 89d9002f8c73efed7f4275f8e0b3257a3d0ea5c7 Mon Sep 17 00:00:00 2001 From: Ricky Moorhouse Date: Sat, 25 Jun 2022 18:25:29 +0100 Subject: [PATCH] feature: monitor CR status & update python version (#34) * feature: monitor CR status * fix: update python version --- Containerfile | 2 +- apiconnect_net.py | 73 +++++++++++++++++++++++++++++++++++++++++++++++ trawler.py | 57 +++++++++++++++++++++++++++++++----- 3 files changed, 124 insertions(+), 8 deletions(-) create mode 100644 apiconnect_net.py diff --git a/Containerfile b/Containerfile index 2423653..16d6877 100644 --- a/Containerfile +++ b/Containerfile @@ -1,5 +1,5 @@ FROM registry.access.redhat.com/ubi8/ubi-minimal -RUN microdnf install -y python3 python3-devel redhat-rpm-config gcc libffi-devel openssl-devel cargo +RUN microdnf install -y python38 python38-devel redhat-rpm-config gcc libffi-devel openssl-devel cargo WORKDIR /app COPY ./requirements.txt /app/requirements.txt RUN python3 -m pip install setuptools_rust diff --git a/apiconnect_net.py b/apiconnect_net.py new file mode 100644 index 0000000..93cd5ce --- /dev/null +++ b/apiconnect_net.py @@ -0,0 +1,73 @@ +import alog +from kubernetes import client, config +import urllib3 + +urllib3.disable_warnings() +logger = alog.use_channel("apiconnect") + +# /mgmt/status/apiconnect/TCPSummary + + +class APIConnectNet(object): + namespace = 'apic-management' + use_kubeconfig = True + trawler = None + + def __init__(self, config, trawler): + # Takes in config object and trawler instance it's behind + # Use kubeconfig or in-cluster config for k8s comms + # self.use_kubeconfig = trawler.use_kubeconfig + # Namespace to find CRs + self.namespace = config.get('namespace', 'default') + self.trawler = trawler + + @alog.timed_function(logger.trace) + def fish(self): + try: + if self.use_kubeconfig: + logger.info("Using KUBECONFIG") + config.load_kube_config() + else: + logger.info("In cluster, so looking for juhu service") + config.load_incluster_config() + customObjectsApi = client.CustomObjectsApi() + customResources = [ + {"group": "management.apiconnect.ibm.com", "plural": "managementclusters"}, + {"group": "gateway.apiconnect.ibm.com", "plural": "gatewayclusters"}, + {"group": "portal.apiconnect.ibm.com", "plural": "portalclusters"}, + ] + for customResource in customResources: + logger.info("Gathering status for {plural}.{group}".format(**customResource)) + api_response = customObjectsApi.list_cluster_custom_object( + customResource['group'], + 'v1beta1', + customResource['plural']) + for item in api_response['items']: + for condition in item['status']['conditions']: + self.trawler.set_gauge( + 'apiconnect', + "{}_status".format(customResource['plural']), + 1, + labels={ + "type": condition['type'], + "status": condition['status'], + "name": item['metadata']['name'], + "namespace": item['metadata']['namespace'], + }) + + # api_response['status']['conditions'] + # {'lastTransitionTime': '2021-09-30T13:33:10Z', 'message': '', 'reason': 'na', 'status': 'False', 'type': 'Warning'} + + except client.rest.ApiException as e: + logger.error("Error calling kubernetes API") + logger.exception(e) + + +if __name__ == "__main__": + import trawler + boaty = trawler.Trawler() + boaty.secret_path = 'test-assets' + boaty.use_kubeconfig = True + net = APIConnectNet({"namespace": "apic"}, boaty) + net.use_kubeconfig = True + net.fish() diff --git a/trawler.py b/trawler.py index 7ce52ca..a0ebbb8 100755 --- a/trawler.py +++ b/trawler.py @@ -9,13 +9,14 @@ import yaml import click from certs_net import CertsNet +from apiconnect_net import APIConnectNet from datapower_net import DataPowerNet from manager_net import ManagerNet from analytics_net import AnalyticsNet from watch_pods import Watcher from prometheus_client import start_http_server import metrics_graphite -from prometheus_client import Gauge +from prometheus_client import Gauge, Counter logger = alog.use_channel("trawler") @@ -42,7 +43,7 @@ def __init__(self, config_file=None): self.load_config(config_file) if 'logging' in self.config: alog.configure( - default_level=self.config['logging'].get('level', 'info'), + default_level=self.config['logging'].get('level', 'debug'), filters=self.config['logging'].get('filters', None), formatter=self.config['logging'].get('format', 'json') ) @@ -88,12 +89,14 @@ def load_config(self, config_file): logger.exception(e) exit(2) - def set_gauge(self, component, target_name, value, pod_name=None, labels={}): + def set_gauge(self, component, target_name, value, pod_name=None, labels=None): + """ Set or create prometheus gauge """ + if not labels: + labels = {} if pod_name: labels['pod'] = pod_name if 'labels' in self.config['prometheus']: - labels = {**self.config['prometheus']['labels'],**labels} - print(labels) + labels = {**self.config['prometheus']['labels'],**labels} logger.debug("Entering set_gauge - params: ({}, {}, {}, {})".format(component, target_name, value, pod_name)) logger.debug(labels) if type(value) is float or type(value) is int: @@ -111,8 +114,8 @@ def set_gauge(self, component, target_name, value, pod_name=None, labels={}): prometheus_target, prometheus_target) - logger.debug("Setting gauge {} to {}".format( - self.gauges[prometheus_target]._name, value)) + logger.debug("Setting gauge %s to %f", + self.gauges[prometheus_target]._name, value) if labels: self.gauges[prometheus_target].labels(**labels).set(value) else: @@ -124,6 +127,44 @@ def set_gauge(self, component, target_name, value, pod_name=None, labels={}): metric_name = "{}.{}".format(component, target_name) self.graphite.stage(metric_name, value) + def inc_counter(self, component, target_name, value, pod_name=None, labels=None): + """ Set or increase prometheus counter """ + if not labels: + labels = {} + if pod_name: + labels['pod'] = pod_name + if 'labels' in self.config['prometheus']: + labels = {**self.config['prometheus']['labels'],**labels} + logger.debug("Entering inc_counter - params: ({}, {}, {}, {})".format(component, target_name, value, pod_name)) + logger.debug(labels) + if type(value) is float or type(value) is int: + target_name = target_name.replace('-', '_') + if self.config['prometheus']['enabled']: + prometheus_target = "{}_{}".format(component, target_name.replace('.', '_')) + if prometheus_target not in self.gauges: + logger.info("Creating counter {}".format(prometheus_target)) + if labels: + self.gauges[prometheus_target] = Counter( + prometheus_target, + prometheus_target, labels.keys()) + else: + self.gauges[prometheus_target] = Counter( + prometheus_target, + prometheus_target) + + logger.debug("Setting gauge %s to %f", + self.gauges[prometheus_target]._name, value) + if labels: + self.gauges[prometheus_target].labels(**labels).inc() + else: + self.gauges[prometheus_target].inc() + if self.config['graphite']['enabled']: + if pod_name: + metric_name = "{}.{}.{}".format(component, pod_name, target_name) + else: + metric_name = "{}.{}".format(component, target_name) + self.graphite.stage(metric_name, value) + @alog.timed_function(logger.trace) def trawl_metrics(self): # Initialise @@ -131,6 +172,8 @@ def trawl_metrics(self): nets = [] if 'certs' in self.config['nets'] and self.config['nets']['certs'].get('enabled', True): nets.append(CertsNet(self.config['nets']['certs'], self)) + if 'apiconnect' in self.config['nets'] and self.config['nets']['apiconnect'].get('enabled', True): + nets.append(APIConnectNet(self.config['nets']['apiconnect'], self)) if 'datapower' in self.config['nets'] and self.config['nets']['datapower'].get('enabled', True): nets.append(DataPowerNet(self.config['nets']['datapower'], self)) if 'manager' in self.config['nets'] and self.config['nets']['manager'].get('enabled', True):