From 873d0fc8baaaa68fd9e7064c1e6cbbdc1b51ccb7 Mon Sep 17 00:00:00 2001 From: Pierre Riteau Date: Mon, 13 Jun 2022 14:18:07 +0200 Subject: [PATCH 1/2] Revert "Support appending custom query suffix" This reverts commit a74847acfd34cd877ae9ce827c96b1cc7c0dc0e3. --- cloudkitty/collector/prometheus.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/cloudkitty/collector/prometheus.py b/cloudkitty/collector/prometheus.py index b85a00d7..769f2fc7 100644 --- a/cloudkitty/collector/prometheus.py +++ b/cloudkitty/collector/prometheus.py @@ -18,7 +18,6 @@ from oslo_config import cfg from oslo_log import log -from voluptuous import All from voluptuous import In from voluptuous import Optional from voluptuous import Required @@ -84,8 +83,7 @@ 'changes', 'delta', 'deriv', 'idelta', 'irange', 'irate', 'rate' - ]), - Optional('query_suffix', default=''): All(str), + ]) } } @@ -161,7 +159,6 @@ def fetch_all(self, metric_name, start, end, scope_id, q_filter=None): 'range_function') groupby = self.conf[metric_name].get('groupby', []) metadata = self.conf[metric_name].get('metadata', []) - query_suffix = self.conf[metric_name]['extra_args']['query_suffix'] period = tzutils.diff_seconds(end, start) time = end @@ -201,10 +198,6 @@ def fetch_all(self, metric_name, start, end, scope_id, q_filter=None): ', '.join(groupby + metadata) ) - # Append custom query suffix - if query_suffix: - query = "{0} {1}".format(query, query_suffix) - LOG.debug("Calling Prometheus with query: %s", query) try: From 7a029941720eb29080a0efe0476a87410991c2c6 Mon Sep 17 00:00:00 2001 From: Pierre Riteau Date: Mon, 10 Jan 2022 15:20:29 +0100 Subject: [PATCH 2/2] Support customising Prometheus queries This commit adds support for adding optional prefix and/or suffix to Prometheus queries. For example, this can be used to perform vector matches between the collected metric and another one, to gather additional metadata. Change-Id: I725f0f5ad00b67f55bcacaf8447e050af3815c73 (cherry picked from commit fe34ef976817f777ddfd49f1ad27925d91305e4a) (cherry picked from commit dc6ab4a4a7fa374f567155e042650a10ee6d49d0) --- cloudkitty/collector/prometheus.py | 15 +++++++++++++-- cloudkitty/tests/collectors/test_validation.py | 8 ++++++++ doc/source/admin/configuration/collector.rst | 6 ++++++ .../prometheus-custom-query-ab2dc00e97b14be2.yaml | 5 +++++ 4 files changed, 32 insertions(+), 2 deletions(-) create mode 100644 releasenotes/notes/prometheus-custom-query-ab2dc00e97b14be2.yaml diff --git a/cloudkitty/collector/prometheus.py b/cloudkitty/collector/prometheus.py index 769f2fc7..8217fef3 100644 --- a/cloudkitty/collector/prometheus.py +++ b/cloudkitty/collector/prometheus.py @@ -18,6 +18,7 @@ from oslo_config import cfg from oslo_log import log +from voluptuous import All from voluptuous import In from voluptuous import Optional from voluptuous import Required @@ -83,7 +84,9 @@ 'changes', 'delta', 'deriv', 'idelta', 'irange', 'irate', 'rate' - ]) + ]), + Optional('query_prefix', default=''): All(str), + Optional('query_suffix', default=''): All(str), } } @@ -159,6 +162,8 @@ def fetch_all(self, metric_name, start, end, scope_id, q_filter=None): 'range_function') groupby = self.conf[metric_name].get('groupby', []) metadata = self.conf[metric_name].get('metadata', []) + query_prefix = self.conf[metric_name]['extra_args']['query_prefix'] + query_suffix = self.conf[metric_name]['extra_args']['query_suffix'] period = tzutils.diff_seconds(end, start) time = end @@ -198,7 +203,13 @@ def fetch_all(self, metric_name, start, end, scope_id, q_filter=None): ', '.join(groupby + metadata) ) - LOG.debug("Calling Prometheus with query: %s", query) + # Add custom query prefix + if query_prefix: + query = "{0} {1}".format(query_prefix, query) + + # Add custom query suffix + if query_suffix: + query = "{0} {1}".format(query, query_suffix) try: res = self._conn.get_instant( diff --git a/cloudkitty/tests/collectors/test_validation.py b/cloudkitty/tests/collectors/test_validation.py index 7a1bdce4..318a6210 100644 --- a/cloudkitty/tests/collectors/test_validation.py +++ b/cloudkitty/tests/collectors/test_validation.py @@ -131,6 +131,8 @@ def test_prometheus_minimal_config_empty_extra_args(self): expected_output['metric_one']['groupby'].append('project_id') expected_output['metric_one']['extra_args'] = { 'aggregation_method': 'max', + 'query_prefix': '', + 'query_suffix': '', } self.assertEqual( collector.prometheus.PrometheusCollector.check_configuration(data), @@ -143,6 +145,8 @@ def test_prometheus_minimal_config_no_extra_args(self): expected_output['metric_one']['groupby'].append('project_id') expected_output['metric_one']['extra_args'] = { 'aggregation_method': 'max', + 'query_prefix': '', + 'query_suffix': '', } self.assertEqual( collector.prometheus.PrometheusCollector.check_configuration(data), @@ -154,6 +158,8 @@ def test_prometheus_minimal_config_minimal_extra_args(self): data['metrics']['metric_one']['extra_args'] = { 'aggregation_method': 'max', 'query_function': 'abs', + 'query_prefix': 'custom_prefix', + 'query_suffix': 'custom_suffix', 'range_function': 'delta', } expected_output = copy.deepcopy(self.base_output) @@ -161,6 +167,8 @@ def test_prometheus_minimal_config_minimal_extra_args(self): expected_output['metric_one']['extra_args'] = { 'aggregation_method': 'max', 'query_function': 'abs', + 'query_prefix': 'custom_prefix', + 'query_suffix': 'custom_suffix', 'range_function': 'delta', } diff --git a/doc/source/admin/configuration/collector.rst b/doc/source/admin/configuration/collector.rst index 5d9fd783..ff316b1d 100644 --- a/doc/source/admin/configuration/collector.rst +++ b/doc/source/admin/configuration/collector.rst @@ -329,6 +329,12 @@ Prometheus ``log10``, ``round``, ``sqrt``. For more information on these functions, you can check `this page`_ +* ``query_prefix``: Optional argument. An arbitrary prefix to add to the + Prometheus query generated by CloudKitty, separated by a space. + +* ``query_suffix``: Optional argument. An arbitrary suffix to add to the + Prometheus query generated by CloudKitty, separated by a space. + * ``range_function``: Optional argument. The function to apply instead of the implicit ``{aggregation_method}_over_time``. Must be one of ``changes``, ``delta``, ``deriv``, ``idelta``, ``irange``, ``irate``, ``rate``. For more diff --git a/releasenotes/notes/prometheus-custom-query-ab2dc00e97b14be2.yaml b/releasenotes/notes/prometheus-custom-query-ab2dc00e97b14be2.yaml new file mode 100644 index 00000000..67477b9b --- /dev/null +++ b/releasenotes/notes/prometheus-custom-query-ab2dc00e97b14be2.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + Adds support for specifying optional prefix and/or suffix to add to + Prometheus queries.