From 0bd70f9014174a4e5a81a963087e3fc4970de345 Mon Sep 17 00:00:00 2001 From: Jacco Flenter Date: Wed, 5 Apr 2023 15:39:48 +0200 Subject: [PATCH 01/23] Add SLO support --- src/autometrics/constants.py | 12 +++++ src/autometrics/decorator.py | 95 +++++++++++++++++++++++++++++------ src/autometrics/objectives.py | 87 ++++++++++++++++++++++++++++++++ 3 files changed, 178 insertions(+), 16 deletions(-) create mode 100644 src/autometrics/constants.py create mode 100644 src/autometrics/objectives.py diff --git a/src/autometrics/constants.py b/src/autometrics/constants.py new file mode 100644 index 0000000..7f97cb5 --- /dev/null +++ b/src/autometrics/constants.py @@ -0,0 +1,12 @@ +COUNTER_DESCRIPTION = "Autometrics counter for tracking function calls" +HISTOGRAM_DESCRIPTION = "Autometrics histogram for tracking function call duration" + +# The following constants are used to create the labels for the prometheus metrics. +# The values are updated to use underscores instead of periods to avoid issues with prometheus. +OBJECTIVE_NAME = "objective.name".replace(".", "_") +OBJECTIVE_PERCENTILE = "objective.percentile".replace(".", "_") +OBJECTIVE_LATENCY_THRESHOLD = "objective.latency_threshold".replace(".", "_") + +OBJECTIVE_NAME_PROMETHEUS = OBJECTIVE_NAME.replace(".", "_") +OBJECTIVE_PERCENTILE_PROMETHEUS = OBJECTIVE_PERCENTILE.replace(".", "_") +OBJECTIVE_LATENCY_THRESHOLD_PROMETHEUS = OBJECTIVE_LATENCY_THRESHOLD.replace(".", "_") diff --git a/src/autometrics/decorator.py b/src/autometrics/decorator.py index be218d8..815a1ee 100644 --- a/src/autometrics/decorator.py +++ b/src/autometrics/decorator.py @@ -1,21 +1,48 @@ -from collections.abc import Callable -from prometheus_client import Counter, Histogram, Gauge -import time +"""Autometrics module.""" import inspect -from .prometheus_url import Generator import os +import time +from typing import Union +from collections.abc import Callable from functools import wraps -from typing import Any, TypeVar +from prometheus_client import Counter, Histogram +from .constants import ( + COUNTER_DESCRIPTION, + HISTOGRAM_DESCRIPTION, + OBJECTIVE_NAME_PROMETHEUS, + OBJECTIVE_PERCENTILE_PROMETHEUS, + OBJECTIVE_LATENCY_THRESHOLD_PROMETHEUS, +) +from .prometheus_url import Generator +from .objectives import Objective prom_counter = Counter( - "function_calls_count", "query??", ["function", "module", "result", "caller"] + "function_calls_count", + COUNTER_DESCRIPTION, + [ + "function", + "module", + "result", + "caller", + OBJECTIVE_NAME_PROMETHEUS, + OBJECTIVE_PERCENTILE_PROMETHEUS, + ], +) +prom_histogram = Histogram( + "function_calls_duration", + HISTOGRAM_DESCRIPTION, + [ + "function", + "module", + OBJECTIVE_NAME_PROMETHEUS, + OBJECTIVE_PERCENTILE_PROMETHEUS, + OBJECTIVE_LATENCY_THRESHOLD_PROMETHEUS, + ], ) -prom_histogram = Histogram("function_calls_duration", "query??", ["function", "module"]) - -R = TypeVar("R") -def autometrics(func: Callable) -> Callable: +def autometrics(func: Callable, objective: Union[None, Objective] = None) -> Callable: + """Decorator for tracking function calls and duration.""" func_name = func.__name__ fullname = func.__qualname__ filename = get_filename_as_module(func) @@ -30,14 +57,44 @@ def wrapper(*args, **kwargs): func_name = func.__name__ start_time = time.time() caller = get_caller_function() + try: result = func(*args, **kwargs) - prom_counter.labels(func_name, module_name, "ok", caller).inc() - except Exception as e: - result = e.__class__.__name__ - prom_counter.labels(func_name, module_name, "error", caller).inc() + prom_counter.labels( + func_name, + module_name, + "ok", + caller, + "" if objective is None else objective.name, + "" + if objective is None or objective.success_rate is None + else objective.success_rate, + ).inc() + except Exception as exception: + result = exception.__class__.__name__ + prom_counter.labels( + func_name, + module_name, + "error", + caller, + "" if objective is None else objective.name, + "" + if objective is None or objective.success_rate is None + else objective.success_rate, + ).inc() + duration = time.time() - start_time - prom_histogram.labels(func_name, module_name).observe(duration) + prom_histogram.labels( + func_name, + module_name, + "" if objective is None else objective.name, + "" + if objective is None or objective.latency is None + else objective.latency[1], + "" + if objective is None or objective.latency is None + else objective.latency[0], + ).observe(duration) return result if func.__doc__ is not None: @@ -48,25 +105,31 @@ def wrapper(*args, **kwargs): def get_filename_as_module(func: Callable) -> str: + """Get the filename of the module that contains the function.""" fullpath = inspect.getsourcefile(func) if fullpath == None: return "" + filename = os.path.basename(fullpath) module_part = os.path.splitext(filename)[0] return module_part def write_docs(func_name: str, module_name: str): + """Write the prometheus query urls to the function docstring.""" g = Generator(func_name, module_name) - urls = g.createURLs() docs = f"Prometheus Query URLs for Function - {func_name} and Module - {module_name}: \n\n" + + urls = g.createURLs() for key, value in urls.items(): docs = f"{docs}{key} : {value} \n\n" + docs = f"{docs}-------------------------------------------\n" return docs def get_caller_function(): + """Get the name of the function that called the function being decorated.""" caller_frame = inspect.stack()[2] caller_function_name = caller_frame[3] return caller_function_name diff --git a/src/autometrics/objectives.py b/src/autometrics/objectives.py new file mode 100644 index 0000000..5eafad5 --- /dev/null +++ b/src/autometrics/objectives.py @@ -0,0 +1,87 @@ +from enum import Enum +from typing import Union, Tuple + + +class ObjectivePercentile(Enum): + """The percentage of requests that must meet the given criteria (success rate or latency).""" + + P90 = "90" + P95 = "95" + P99 = "99" + P99_9 = "99.9" + + +class ObjectiveLatency(Enum): + """The latency threshold for the given percentile.""" + + Ms5 = "0.005" + Ms10 = "0.01" + Ms25 = "0.025" + Ms50 = "0.05" + Ms75 = "0.075" + Ms100 = "0.1" + Ms250 = "0.25" + Ms500 = "0.5" + Ms750 = "0.75" + Ms1000 = "1" + Ms2500 = "2.5" + Ms5000 = "5" + Ms7500 = "7.5" + Ms10000 = "10" + + +# This represents a Service-Level Objective (SLO) for a function or group of functions. +# The objective should be given a descriptive name and can represent +# a success rate and/or latency objective. +# +# For details on SLOs, see +# +# Example: +# ```python +# from autometrics.autometrics import autometrics +# from autometrics.objective import Objective, ObjectivePercentile, TargetLatency +# API_SLO: = Objective("api", success_rate=ObjectivePercentile.P99_9, latency = (TargetLatency.Ms250, ObjectivePercentile.P99)) +# +# @autometrics(objective = API_SLO)] +# def api_handler() : +# # ... +# ``` +# +# ## How this works +# +# When an objective is added to a function, the metrics for that function will +# have additional labels attached to specify the SLO details. +# +# Autometrics comes with a set of Prometheus [recording rules](https://prometheus.io/docs/prometheus/latest/configuration/recording_rules/) +# and [alerting rules](https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/) +# that will fire alerts when the given objective is being violated. +# +# By default, these recording rules will effectively lay dormant. +# However, they are enabled when the special labels are present on certain metrics. +class Objective: + """A Service-Level Objective (SLO) for a function or group of functions.""" + + name: str + """name: The name of the objective. This should be something descriptive of the function or group of functions it covers.""" + success_rate: Union[None, ObjectivePercentile] + """Specify the success rate for this objective. + + This means that the function or group of functions that are part of this objective + should return an `Ok` result at least this percentage of the time.""" + latency = Union[None, Tuple[ObjectiveLatency, ObjectivePercentile]] + + def __init__( + self, + name: str, + success_rate: Union[None, ObjectivePercentile] = None, + latency: Union[None, Tuple[ObjectiveLatency, ObjectivePercentile]] = None, + ): + """Create a new objective with the given name. + + The name should be something descriptive of the function or group of functions it covers. + For example, if you have an objective covering all of the HTTP handlers in your API you might call it "api". + """ + + self.name = name + self.success_rate = success_rate + self.latency = latency From 3f0b2c4fae39b6769df1e0172aaedb7809ecdc89 Mon Sep 17 00:00:00 2001 From: Jacco Flenter Date: Thu, 6 Apr 2023 11:52:09 +0200 Subject: [PATCH 02/23] Refactor wrapper Extract out prometheus related logic --- src/autometrics/decorator.py | 99 ++++++++---------------------------- src/autometrics/emit.py | 73 ++++++++++++++++++++++++++ 2 files changed, 95 insertions(+), 77 deletions(-) create mode 100644 src/autometrics/emit.py diff --git a/src/autometrics/decorator.py b/src/autometrics/decorator.py index 815a1ee..766ff39 100644 --- a/src/autometrics/decorator.py +++ b/src/autometrics/decorator.py @@ -5,109 +5,54 @@ from typing import Union from collections.abc import Callable from functools import wraps -from prometheus_client import Counter, Histogram -from .constants import ( - COUNTER_DESCRIPTION, - HISTOGRAM_DESCRIPTION, - OBJECTIVE_NAME_PROMETHEUS, - OBJECTIVE_PERCENTILE_PROMETHEUS, - OBJECTIVE_LATENCY_THRESHOLD_PROMETHEUS, -) from .prometheus_url import Generator from .objectives import Objective - -prom_counter = Counter( - "function_calls_count", - COUNTER_DESCRIPTION, - [ - "function", - "module", - "result", - "caller", - OBJECTIVE_NAME_PROMETHEUS, - OBJECTIVE_PERCENTILE_PROMETHEUS, - ], -) -prom_histogram = Histogram( - "function_calls_duration", - HISTOGRAM_DESCRIPTION, - [ - "function", - "module", - OBJECTIVE_NAME_PROMETHEUS, - OBJECTIVE_PERCENTILE_PROMETHEUS, - OBJECTIVE_LATENCY_THRESHOLD_PROMETHEUS, - ], -) +from .emit import count, histogram def autometrics(func: Callable, objective: Union[None, Objective] = None) -> Callable: """Decorator for tracking function calls and duration.""" + module_name = get_module_name(func) func_name = func.__name__ - fullname = func.__qualname__ - filename = get_filename_as_module(func) - if fullname == func_name: - module_name = filename - else: - classname = func.__qualname__.rsplit(".", 1)[0] - module_name = f"{filename}.{classname}" @wraps(func) def wrapper(*args, **kwargs): - func_name = func.__name__ start_time = time.time() caller = get_caller_function() try: result = func(*args, **kwargs) - prom_counter.labels( - func_name, - module_name, - "ok", - caller, - "" if objective is None else objective.name, - "" - if objective is None or objective.success_rate is None - else objective.success_rate, - ).inc() + count(func_name, module_name, caller, objective, "ok") except Exception as exception: result = exception.__class__.__name__ - prom_counter.labels( - func_name, - module_name, - "error", - caller, - "" if objective is None else objective.name, - "" - if objective is None or objective.success_rate is None - else objective.success_rate, - ).inc() - - duration = time.time() - start_time - prom_histogram.labels( - func_name, - module_name, - "" if objective is None else objective.name, - "" - if objective is None or objective.latency is None - else objective.latency[1], - "" - if objective is None or objective.latency is None - else objective.latency[0], - ).observe(duration) + count(func_name, module_name, caller, objective, "error") + + histogram(func_name, module_name, start_time, objective) return result - if func.__doc__ is not None: - wrapper.__doc__ = f"{func.__doc__}\n{write_docs(func_name, module_name)}" - else: + if func.__doc__ is None: wrapper.__doc__ = write_docs(func_name, module_name) + else: + wrapper.__doc__ = f"{func.__doc__}\n{write_docs(func_name, module_name)}" return wrapper +def get_module_name(func: Callable) -> str: + """Get the name of the module that contains the function.""" + func_name = func.__name__ + fullname = func.__qualname__ + filename = get_filename_as_module(func) + if fullname == func_name: + return filename + + classname = func.__qualname__.rsplit(".", 1)[0] + return f"{filename}.{classname}" + + def get_filename_as_module(func: Callable) -> str: """Get the filename of the module that contains the function.""" fullpath = inspect.getsourcefile(func) - if fullpath == None: + if fullpath is None: return "" filename = os.path.basename(fullpath) diff --git a/src/autometrics/emit.py b/src/autometrics/emit.py new file mode 100644 index 0000000..32b0a0d --- /dev/null +++ b/src/autometrics/emit.py @@ -0,0 +1,73 @@ +import time +from typing import Union +from prometheus_client import Counter, Histogram + +from .constants import ( + COUNTER_DESCRIPTION, + HISTOGRAM_DESCRIPTION, + OBJECTIVE_NAME_PROMETHEUS, + OBJECTIVE_PERCENTILE_PROMETHEUS, + OBJECTIVE_LATENCY_THRESHOLD_PROMETHEUS, +) +from .objectives import Objective + +prom_counter = Counter( + "function_calls_count", + COUNTER_DESCRIPTION, + [ + "function", + "module", + "result", + "caller", + OBJECTIVE_NAME_PROMETHEUS, + OBJECTIVE_PERCENTILE_PROMETHEUS, + ], +) +prom_histogram = Histogram( + "function_calls_duration", + HISTOGRAM_DESCRIPTION, + [ + "function", + "module", + OBJECTIVE_NAME_PROMETHEUS, + OBJECTIVE_PERCENTILE_PROMETHEUS, + OBJECTIVE_LATENCY_THRESHOLD_PROMETHEUS, + ], +) + + +def count( + func_name: str, + module_name: str, + caller: str, + objective: Union[None, Objective] = None, + result: Union("error", "ok") = "ok", +): + """Increment the counter for the function call.""" + prom_counter.labels( + func_name, + module_name, + result, + caller, + "" if objective is None else objective.name, + "" + if objective is None or objective.success_rate is None + else objective.success_rate, + ).inc() + + +def histogram( + func_name: str, + module_name: str, + start_time: float, + objective: Union[None, Objective] = None, +): + """Observe the duration of the function call.""" + duration = time.time() - start_time + prom_histogram.labels( + func_name, + module_name, + "" if objective is None else objective.name, + "" if objective is None or objective.latency is None else objective.latency[1], + "" if objective is None or objective.latency is None else objective.latency[0], + ).observe(duration) From 4f205553720957ac9cdc21caf3e92534563ce9be Mon Sep 17 00:00:00 2001 From: Jacco Flenter Date: Thu, 6 Apr 2023 12:06:11 +0200 Subject: [PATCH 03/23] refactor prometheus code generator To better follow pyton conventions --- src/autometrics/decorator.py | 4 ++-- src/autometrics/prometheus_url.py | 37 +++++++++++++++++-------------- src/test_prometheus_url.py | 10 ++++----- 3 files changed, 27 insertions(+), 24 deletions(-) diff --git a/src/autometrics/decorator.py b/src/autometrics/decorator.py index 766ff39..8e6853f 100644 --- a/src/autometrics/decorator.py +++ b/src/autometrics/decorator.py @@ -62,10 +62,10 @@ def get_filename_as_module(func: Callable) -> str: def write_docs(func_name: str, module_name: str): """Write the prometheus query urls to the function docstring.""" - g = Generator(func_name, module_name) + generator = Generator(func_name, module_name) docs = f"Prometheus Query URLs for Function - {func_name} and Module - {module_name}: \n\n" - urls = g.createURLs() + urls = generator.create_urls() for key, value in urls.items(): docs = f"{docs}{key} : {value} \n\n" diff --git a/src/autometrics/prometheus_url.py b/src/autometrics/prometheus_url.py index 96e4bbd..2ba1208 100644 --- a/src/autometrics/prometheus_url.py +++ b/src/autometrics/prometheus_url.py @@ -4,35 +4,38 @@ class Generator: - def __init__(self, functionName, moduleName, baseUrl=None): + def __init__(self, function_name: str, module_name: str, base_url: str=None): load_dotenv() - self.functionName = functionName - self.moduleName = moduleName - self.baseUrl = baseUrl or os.getenv("PROMETHEUS_URL") - if self.baseUrl is None: - self.baseUrl = "http://localhost:9090" - elif self.baseUrl[-1] == "/": - self.baseUrl = self.baseUrl[ + self.function_name = function_name + self.module_name = module_name + self.baseUrl = base_url or os.getenv("PROMETHEUS_URL") + self.base_url = os.getenv("PROMETHEUS_URL") + + if self.base_url is None: + self.base_url = "http://localhost:9090" + elif self.base_url[-1] == "/": + self.base_url = self.base_url[ :-1 ] # Remove the trailing slash if there is one - def createURLs(self): - requestRateQuery = f'sum by (function, module) (rate (function_calls_count_total{{function="{self.functionName}",module="{self.moduleName}"}}[5m]))' - latencyQuery = f'sum by (le, function, module) (rate(function_calls_duration_bucket{{function="{self.functionName}",module="{self.moduleName}"}}[5m]))' - errorRatioQuery = f'sum by (function, module) (rate (function_calls_count_total{{function="{self.functionName}",module="{self.moduleName}", result="error"}}[5m])) / {requestRateQuery}' + def create_urls(self): + """Create the prometheus query urls for the function and module.""" + request_rate_query = f'sum by (function, module) (rate (function_calls_count_total{{function="{self.function_name}",module="{self.module_name}"}}[5m]))' + latency_query = f'sum by (le, function, module) (rate(function_calls_duration_bucket{{function="{self.function_name}",module="{self.module_name}"}}[5m]))' + error_ratio_query = f'sum by (function, module) (rate (function_calls_count_total{{function="{self.function_name}",module="{self.module_name}", result="error"}}[5m])) / {request_rate_query}' - queries = [requestRateQuery, latencyQuery, errorRatioQuery] + queries = [request_rate_query, latency_query, error_ratio_query] names = ["Request rate URL", "Latency URL", "Error Ratio URL"] urls = {} - for n in names: + for name in names: for query in queries: - generateUrl = self.createPrometheusUrl(query) - urls[n] = generateUrl + generated_url = self.create_prometheus_url(query) + urls[name] = generated_url queries.remove(query) break return urls - def createPrometheusUrl(self, query): + def create_prometheus_url(self, query): urlEncode = urllib.parse.quote(query) url = f"{self.baseUrl}/graph?g0.expr={urlEncode}&g0.tab=0" return url diff --git a/src/test_prometheus_url.py b/src/test_prometheus_url.py index d9b7f65..d06991e 100644 --- a/src/test_prometheus_url.py +++ b/src/test_prometheus_url.py @@ -7,8 +7,8 @@ class TestPrometheusUrlGeneratorDefault(unittest.TestCase): def setUp(self): self.generator = Generator("myFunction", "myModule") - def test_createPrometheusUrl(self): - url = self.generator.createPrometheusUrl("myQuery") + def test_create_prometheus_url(self): + url = self.generator.create_prometheus_url("myQuery") self.assertTrue( url.startswith("http://localhost:9090/graph?g0.expr=") ) # Make sure the base URL is correct @@ -19,11 +19,11 @@ def test_createPrometheusUrl(self): class TestPrometheusUrlGeneratorCustomUrl(unittest.TestCase): def setUp(self): self.generator = Generator( - "myFunction", "myModule", baseUrl="http://localhost:9091" + "myFunction", "myModule", base_url="http://localhost:9091" ) - def test_createPrometheusUrl(self): - url = self.generator.createPrometheusUrl("myQuery") + def test_create_prometheus_url(self): + url = self.generator.create_prometheus_url("myQuery") self.assertTrue( url.startswith("http://localhost:9091/graph?g0.expr=") ) # Make sure the base URL is correct From 3577624d09f77368b27ae5e232be872ae85bdb35 Mon Sep 17 00:00:00 2001 From: Jacco Flenter Date: Thu, 6 Apr 2023 15:06:56 +0200 Subject: [PATCH 04/23] Make decorator more flexible to support objective As a parameter --- examples/example.py | 23 ++++++- src/autometrics/decorator.py | 112 ++++++++++++++-------------------- src/autometrics/emit.py | 10 ++- src/autometrics/objectives.py | 4 +- src/autometrics/utils.py | 47 ++++++++++++++ 5 files changed, 125 insertions(+), 71 deletions(-) create mode 100644 src/autometrics/utils.py diff --git a/examples/example.py b/examples/example.py index ce2384f..e777f3e 100644 --- a/examples/example.py +++ b/examples/example.py @@ -1,7 +1,8 @@ -from prometheus_client import start_http_server -from autometrics import autometrics import time import random +from prometheus_client import start_http_server +from autometrics import autometrics +from autometrics.objectives import Objective, ObjectiveLatency, ObjectivePercentile # Defines a class called `Operations`` that has two methods: @@ -35,6 +36,23 @@ def div_unhandled(num1, num2): result = num1 / num2 return result +RANDOM_SLO = Objective( + "api", + success_rate=ObjectivePercentile.P99_9, + latency=(ObjectiveLatency.Ms250, ObjectivePercentile.P99), +) + + +@autometrics(objective=RANDOM_SLO) +def random_error(): + """This function will randomly return an error or not.""" + + result = random.choice(["ok", "error"]) + if result == "error": + time.sleep(1) + raise Exception("random error") + return result + ops = Operations() @@ -59,3 +77,4 @@ def div_unhandled(num1, num2): time.sleep(2) # Call `div_unhandled` such that it raises an error div_unhandled(2, 0) + random_error() diff --git a/src/autometrics/decorator.py b/src/autometrics/decorator.py index 8e6853f..7910ee0 100644 --- a/src/autometrics/decorator.py +++ b/src/autometrics/decorator.py @@ -1,80 +1,60 @@ """Autometrics module.""" -import inspect -import os import time -from typing import Union -from collections.abc import Callable +from typing import Union, overload, TypeVar, Any, Callable, Optional from functools import wraps -from .prometheus_url import Generator from .objectives import Objective from .emit import count, histogram +from .utils import get_module_name, get_caller_function, write_docs -def autometrics(func: Callable, objective: Union[None, Objective] = None) -> Callable: - """Decorator for tracking function calls and duration.""" - module_name = get_module_name(func) - func_name = func.__name__ - - @wraps(func) - def wrapper(*args, **kwargs): - start_time = time.time() - caller = get_caller_function() - - try: - result = func(*args, **kwargs) - count(func_name, module_name, caller, objective, "ok") - except Exception as exception: - result = exception.__class__.__name__ - count(func_name, module_name, caller, objective, "error") - - histogram(func_name, module_name, start_time, objective) - return result - - if func.__doc__ is None: - wrapper.__doc__ = write_docs(func_name, module_name) - else: - wrapper.__doc__ = f"{func.__doc__}\n{write_docs(func_name, module_name)}" - return wrapper - +F = TypeVar("F", bound=Callable[..., Any]) -def get_module_name(func: Callable) -> str: - """Get the name of the module that contains the function.""" - func_name = func.__name__ - fullname = func.__qualname__ - filename = get_filename_as_module(func) - if fullname == func_name: - return filename - classname = func.__qualname__.rsplit(".", 1)[0] - return f"{filename}.{classname}" +# Bare decorator usage +@overload +def autometrics(func: F) -> F: + ... -def get_filename_as_module(func: Callable) -> str: - """Get the filename of the module that contains the function.""" - fullpath = inspect.getsourcefile(func) - if fullpath is None: - return "" +# Decorator with arguments +@overload +def autometrics(*, objective: Union[None, Objective] = None) -> Callable[[F], F]: + ... - filename = os.path.basename(fullpath) - module_part = os.path.splitext(filename)[0] - return module_part - - -def write_docs(func_name: str, module_name: str): - """Write the prometheus query urls to the function docstring.""" - generator = Generator(func_name, module_name) - docs = f"Prometheus Query URLs for Function - {func_name} and Module - {module_name}: \n\n" - - urls = generator.create_urls() - for key, value in urls.items(): - docs = f"{docs}{key} : {value} \n\n" - - docs = f"{docs}-------------------------------------------\n" - return docs +def autometrics( + func: Optional[Callable[..., Any]] = None, + *, + objective: Union[None, Objective] = None, +): + """Decorator for tracking function calls and duration.""" -def get_caller_function(): - """Get the name of the function that called the function being decorated.""" - caller_frame = inspect.stack()[2] - caller_function_name = caller_frame[3] - return caller_function_name + def decorator(func: F) -> F: + module_name = get_module_name(func) + func_name = func.__name__ + + @wraps(func) + def wrapper(*args, **kwargs): + start_time = time.time() + caller = get_caller_function() + + try: + result = func(*args, **kwargs) + count(func_name, module_name, caller, objective, "ok") + except Exception as exception: + result = exception.__class__.__name__ + count(func_name, module_name, caller, objective, "error") + + histogram(func_name, module_name, start_time, objective) + return result + + if func.__doc__ is None: + wrapper.__doc__ = write_docs(func_name, module_name) + else: + wrapper.__doc__ = f"{func.__doc__}\n{write_docs(func_name, module_name)}" + return wrapper + + if func is None: + return decorator + else: + return decorator(func) diff --git a/src/autometrics/emit.py b/src/autometrics/emit.py index 32b0a0d..74df290 100644 --- a/src/autometrics/emit.py +++ b/src/autometrics/emit.py @@ -1,4 +1,5 @@ import time +from enum import Enum from typing import Union from prometheus_client import Counter, Histogram @@ -36,12 +37,19 @@ ) +class Result(Enum): + """Result of the function call.""" + + OK = "ok" + ERROR = "error" + + def count( func_name: str, module_name: str, caller: str, objective: Union[None, Objective] = None, - result: Union("error", "ok") = "ok", + result: Result = Result.OK, ): """Increment the counter for the function call.""" prom_counter.labels( diff --git a/src/autometrics/objectives.py b/src/autometrics/objectives.py index 5eafad5..28ca013 100644 --- a/src/autometrics/objectives.py +++ b/src/autometrics/objectives.py @@ -39,8 +39,8 @@ class ObjectiveLatency(Enum): # Example: # ```python # from autometrics.autometrics import autometrics -# from autometrics.objective import Objective, ObjectivePercentile, TargetLatency -# API_SLO: = Objective("api", success_rate=ObjectivePercentile.P99_9, latency = (TargetLatency.Ms250, ObjectivePercentile.P99)) +# from autometrics.objectives import Objective, ObjectivePercentile, ObjectiveLatency +# API_SLO = Objective("api", success_rate=ObjectivePercentile.P99_9, latency = (ObjectiveLatency.Ms250, ObjectivePercentile.P99)) # # @autometrics(objective = API_SLO)] # def api_handler() : diff --git a/src/autometrics/utils.py b/src/autometrics/utils.py new file mode 100644 index 0000000..dc6258c --- /dev/null +++ b/src/autometrics/utils.py @@ -0,0 +1,47 @@ +import inspect +import os +from collections.abc import Callable +from .prometheus_url import Generator + + +def get_module_name(func: Callable) -> str: + """Get the name of the module that contains the function.""" + func_name = func.__name__ + fullname = func.__qualname__ + filename = get_filename_as_module(func) + if fullname == func_name: + return filename + + classname = func.__qualname__.rsplit(".", 1)[0] + return f"{filename}.{classname}" + + +def get_filename_as_module(func: Callable) -> str: + """Get the filename of the module that contains the function.""" + fullpath = inspect.getsourcefile(func) + if fullpath is None: + return "" + + filename = os.path.basename(fullpath) + module_part = os.path.splitext(filename)[0] + return module_part + + +def write_docs(func_name: str, module_name: str): + """Write the prometheus query urls to the function docstring.""" + generator = Generator(func_name, module_name) + docs = f"Prometheus Query URLs for Function - {func_name} and Module - {module_name}: \n\n" + + urls = generator.create_urls() + for key, value in urls.items(): + docs = f"{docs}{key} : {value} \n\n" + + docs = f"{docs}-------------------------------------------\n" + return docs + + +def get_caller_function(): + """Get the name of the function that called the function being decorated.""" + caller_frame = inspect.stack()[2] + caller_function_name = caller_frame[3] + return caller_function_name From ea73a55251ee81b7061f4aac4756d4a20c99e754 Mon Sep 17 00:00:00 2001 From: Jacco Flenter Date: Thu, 6 Apr 2023 16:06:09 +0200 Subject: [PATCH 05/23] Update comment in constants --- src/autometrics/constants.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/autometrics/constants.py b/src/autometrics/constants.py index 7f97cb5..d089c89 100644 --- a/src/autometrics/constants.py +++ b/src/autometrics/constants.py @@ -1,12 +1,15 @@ +"""Constants used by autometrics""" + COUNTER_DESCRIPTION = "Autometrics counter for tracking function calls" HISTOGRAM_DESCRIPTION = "Autometrics histogram for tracking function call duration" -# The following constants are used to create the labels for the prometheus metrics. -# The values are updated to use underscores instead of periods to avoid issues with prometheus. +# The following constants are used to create the labels OBJECTIVE_NAME = "objective.name".replace(".", "_") OBJECTIVE_PERCENTILE = "objective.percentile".replace(".", "_") OBJECTIVE_LATENCY_THRESHOLD = "objective.latency_threshold".replace(".", "_") +# The values are updated to use underscores instead of periods to avoid issues with prometheus. +# A similar thing is done in the rust library, which supports multiple exporters OBJECTIVE_NAME_PROMETHEUS = OBJECTIVE_NAME.replace(".", "_") OBJECTIVE_PERCENTILE_PROMETHEUS = OBJECTIVE_PERCENTILE.replace(".", "_") OBJECTIVE_LATENCY_THRESHOLD_PROMETHEUS = OBJECTIVE_LATENCY_THRESHOLD.replace(".", "_") From 0f5ff633a38d2ff046e8f8d1cdda50f1335ee3e9 Mon Sep 17 00:00:00 2001 From: Jacco Flenter Date: Thu, 6 Apr 2023 16:08:41 +0200 Subject: [PATCH 06/23] Add more docstrings + minor refactor --- src/autometrics/prometheus_url.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/autometrics/prometheus_url.py b/src/autometrics/prometheus_url.py index 2ba1208..dc9b495 100644 --- a/src/autometrics/prometheus_url.py +++ b/src/autometrics/prometheus_url.py @@ -35,7 +35,8 @@ def create_urls(self): break return urls - def create_prometheus_url(self, query): - urlEncode = urllib.parse.quote(query) - url = f"{self.baseUrl}/graph?g0.expr={urlEncode}&g0.tab=0" + def create_prometheus_url(self, query: str): + """Create a the full query url for a given query.""" + encoded_query = urllib.parse.quote(query) + url = f"{self.baseUrl}/graph?g0.expr={encoded_query}&g0.tab=0" return url From 56f418ee9e7cb6eee1658287d9a71ba38f087391 Mon Sep 17 00:00:00 2001 From: Jacco Flenter Date: Thu, 6 Apr 2023 16:11:56 +0200 Subject: [PATCH 07/23] Fix formatting --- examples/example.py | 2 +- src/autometrics/constants.py | 2 +- src/autometrics/objectives.py | 8 ++++++-- src/autometrics/prometheus_url.py | 5 +++-- 4 files changed, 11 insertions(+), 6 deletions(-) diff --git a/examples/example.py b/examples/example.py index e777f3e..1f44435 100644 --- a/examples/example.py +++ b/examples/example.py @@ -37,7 +37,7 @@ def div_unhandled(num1, num2): return result RANDOM_SLO = Objective( - "api", + "random", success_rate=ObjectivePercentile.P99_9, latency=(ObjectiveLatency.Ms250, ObjectivePercentile.P99), ) diff --git a/src/autometrics/constants.py b/src/autometrics/constants.py index d089c89..14c6fbd 100644 --- a/src/autometrics/constants.py +++ b/src/autometrics/constants.py @@ -3,7 +3,7 @@ COUNTER_DESCRIPTION = "Autometrics counter for tracking function calls" HISTOGRAM_DESCRIPTION = "Autometrics histogram for tracking function call duration" -# The following constants are used to create the labels +# The following constants are used to create the labels OBJECTIVE_NAME = "objective.name".replace(".", "_") OBJECTIVE_PERCENTILE = "objective.percentile".replace(".", "_") OBJECTIVE_LATENCY_THRESHOLD = "objective.latency_threshold".replace(".", "_") diff --git a/src/autometrics/objectives.py b/src/autometrics/objectives.py index 28ca013..f470148 100644 --- a/src/autometrics/objectives.py +++ b/src/autometrics/objectives.py @@ -38,9 +38,13 @@ class ObjectiveLatency(Enum): # # Example: # ```python -# from autometrics.autometrics import autometrics +# from autometrics import autometrics # from autometrics.objectives import Objective, ObjectivePercentile, ObjectiveLatency -# API_SLO = Objective("api", success_rate=ObjectivePercentile.P99_9, latency = (ObjectiveLatency.Ms250, ObjectivePercentile.P99)) +# API_SLO = Objective( +# "api", +# success_rate=ObjectivePercentile.P99_9, +# latency=(ObjectiveLatency.Ms250, ObjectivePercentile.P99), +# ) # # @autometrics(objective = API_SLO)] # def api_handler() : diff --git a/src/autometrics/prometheus_url.py b/src/autometrics/prometheus_url.py index dc9b495..413616c 100644 --- a/src/autometrics/prometheus_url.py +++ b/src/autometrics/prometheus_url.py @@ -4,12 +4,13 @@ class Generator: + """Generate prometheus query urls for a given function/module.""" + def __init__(self, function_name: str, module_name: str, base_url: str=None): load_dotenv() self.function_name = function_name self.module_name = module_name - self.baseUrl = base_url or os.getenv("PROMETHEUS_URL") - self.base_url = os.getenv("PROMETHEUS_URL") + self.base_url = base_url or os.getenv("PROMETHEUS_URL") if self.base_url is None: self.base_url = "http://localhost:9090" From 54750a9222cd80c518d93751db15edda15ae8d94 Mon Sep 17 00:00:00 2001 From: Jacco Flenter Date: Thu, 6 Apr 2023 16:56:00 +0200 Subject: [PATCH 08/23] Fix linting issues --- README.md | 9 +++++---- src/autometrics/decorator.py | 31 +++++++++++++++++-------------- src/autometrics/emit.py | 35 +++++++++++++++++++++++++---------- 3 files changed, 47 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index ce72840..3178c1e 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ See [Why Autometrics?](https://github.com/autometrics-dev#why-autometrics) for m - 🔗 Create links to live Prometheus charts directly into each functions docstrings (with tooltips coming soon!) - 📊 (Coming Soon!) Grafana dashboard showing the performance of all instrumented functions -- 🚨 (Coming Soon!) Generates Prometheus alerting rules using SLO best practices +- 🚨 Emits data for Prometheus alerting rules using SLO best practices from simple annotations in your code - ⚡ Minimal runtime overhead @@ -48,9 +48,10 @@ def sayHello: ## Development of the package This package uses [poetry](https://python-poetry.org) as a package manager, with all dependencies separated into three groups: - - root level dependencies, required - - `dev`, everything that is needed for development or in ci - - `examples`, dependencies of everything in `examples/` directory + +- root level dependencies, required +- `dev`, everything that is needed for development or in ci +- `examples`, dependencies of everything in `examples/` directory By default, poetry will only install required dependencies, if you want to run examples, install using this command: diff --git a/src/autometrics/decorator.py b/src/autometrics/decorator.py index 7910ee0..48d46dd 100644 --- a/src/autometrics/decorator.py +++ b/src/autometrics/decorator.py @@ -1,51 +1,54 @@ """Autometrics module.""" import time from typing import Union, overload, TypeVar, Any, Callable, Optional +from typing_extensions import ParamSpec from functools import wraps from .objectives import Objective -from .emit import count, histogram +from .emit import count, histogram, Result from .utils import get_module_name, get_caller_function, write_docs -F = TypeVar("F", bound=Callable[..., Any]) +P = ParamSpec("P") +T = TypeVar("T") # Bare decorator usage @overload -def autometrics(func: F) -> F: +def autometrics(func: Callable[P, T]) -> Callable[P, T]: ... # Decorator with arguments @overload -def autometrics(*, objective: Union[None, Objective] = None) -> Callable[[F], F]: +def autometrics(*, objective: Optional[Objective] = None) -> Callable: ... - def autometrics( - func: Optional[Callable[..., Any]] = None, + func: Optional[Callable] = None, *, - objective: Union[None, Objective] = None, + objective: Optional[Objective] = None, ): """Decorator for tracking function calls and duration.""" - def decorator(func: F) -> F: + def decorator(func: Callable[P, T]) -> Callable[P, T]: module_name = get_module_name(func) func_name = func.__name__ @wraps(func) - def wrapper(*args, **kwargs): + def wrapper(*args: P.args, **kwds: P.kwargs) -> T: start_time = time.time() caller = get_caller_function() try: - result = func(*args, **kwargs) - count(func_name, module_name, caller, objective, "ok") + result = func(*args, **kwds) + count(func_name, module_name, caller, objective, result=Result.OK) + histogram(func_name, module_name, start_time, objective) except Exception as exception: result = exception.__class__.__name__ - count(func_name, module_name, caller, objective, "error") - - histogram(func_name, module_name, start_time, objective) + count(func_name, module_name, caller, objective, result=Result.ERROR) + histogram(func_name, module_name, start_time, objective) + # Reraise exception + raise exception return result if func.__doc__ is None: diff --git a/src/autometrics/emit.py b/src/autometrics/emit.py index 74df290..d6cc21e 100644 --- a/src/autometrics/emit.py +++ b/src/autometrics/emit.py @@ -1,6 +1,6 @@ import time from enum import Enum -from typing import Union +from typing import Optional from prometheus_client import Counter, Histogram from .constants import ( @@ -48,19 +48,25 @@ def count( func_name: str, module_name: str, caller: str, - objective: Union[None, Objective] = None, + objective: Optional[Objective] = None, result: Result = Result.OK, ): """Increment the counter for the function call.""" + + objective_name = "" if objective is None else objective.name + percentile = ( + "" + if objective is None or objective.success_rate is None + else objective.success_rate + ) + prom_counter.labels( func_name, module_name, result, caller, - "" if objective is None else objective.name, - "" - if objective is None or objective.success_rate is None - else objective.success_rate, + objective_name, + percentile, ).inc() @@ -68,14 +74,23 @@ def histogram( func_name: str, module_name: str, start_time: float, - objective: Union[None, Objective] = None, + objective: Optional[Objective] = None, ): """Observe the duration of the function call.""" duration = time.time() - start_time + + objective_name = "" if objective is None else objective.name + latency = None if objective is None else objective.latency + percentile = "" + threshold = "" + if latency is not None: + threshold = latency[0] + percentile = latency[1] + prom_histogram.labels( func_name, module_name, - "" if objective is None else objective.name, - "" if objective is None or objective.latency is None else objective.latency[1], - "" if objective is None or objective.latency is None else objective.latency[0], + objective_name, + percentile, + threshold, ).observe(duration) From 0e583b41f46bbdcfe3763bb2aca1f819eb773e83 Mon Sep 17 00:00:00 2001 From: Jacco Flenter Date: Thu, 6 Apr 2023 17:08:39 +0200 Subject: [PATCH 09/23] Format code --- examples/example.py | 1 + src/autometrics/decorator.py | 1 + src/autometrics/emit.py | 2 +- src/autometrics/prometheus_url.py | 4 ++-- 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/examples/example.py b/examples/example.py index 1f44435..bcbd657 100644 --- a/examples/example.py +++ b/examples/example.py @@ -36,6 +36,7 @@ def div_unhandled(num1, num2): result = num1 / num2 return result + RANDOM_SLO = Objective( "random", success_rate=ObjectivePercentile.P99_9, diff --git a/src/autometrics/decorator.py b/src/autometrics/decorator.py index 48d46dd..fca8e90 100644 --- a/src/autometrics/decorator.py +++ b/src/autometrics/decorator.py @@ -23,6 +23,7 @@ def autometrics(func: Callable[P, T]) -> Callable[P, T]: def autometrics(*, objective: Optional[Objective] = None) -> Callable: ... + def autometrics( func: Optional[Callable] = None, *, diff --git a/src/autometrics/emit.py b/src/autometrics/emit.py index d6cc21e..8dbe8d5 100644 --- a/src/autometrics/emit.py +++ b/src/autometrics/emit.py @@ -83,7 +83,7 @@ def histogram( latency = None if objective is None else objective.latency percentile = "" threshold = "" - if latency is not None: + if latency is not None: threshold = latency[0] percentile = latency[1] diff --git a/src/autometrics/prometheus_url.py b/src/autometrics/prometheus_url.py index 413616c..86c85ba 100644 --- a/src/autometrics/prometheus_url.py +++ b/src/autometrics/prometheus_url.py @@ -6,12 +6,12 @@ class Generator: """Generate prometheus query urls for a given function/module.""" - def __init__(self, function_name: str, module_name: str, base_url: str=None): + def __init__(self, function_name: str, module_name: str, base_url: str = None): load_dotenv() self.function_name = function_name self.module_name = module_name self.base_url = base_url or os.getenv("PROMETHEUS_URL") - + if self.base_url is None: self.base_url = "http://localhost:9090" elif self.base_url[-1] == "/": From 1628615370a13535e8617d52bb83349897909d25 Mon Sep 17 00:00:00 2001 From: Jacco Flenter Date: Tue, 11 Apr 2023 09:51:39 +0200 Subject: [PATCH 10/23] Fix typing complaints --- src/autometrics/objectives.py | 10 +++++----- src/autometrics/prometheus_url.py | 23 ++++++++++++++--------- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/src/autometrics/objectives.py b/src/autometrics/objectives.py index f470148..7291cd0 100644 --- a/src/autometrics/objectives.py +++ b/src/autometrics/objectives.py @@ -1,5 +1,5 @@ from enum import Enum -from typing import Union, Tuple +from typing import Optional, Tuple class ObjectivePercentile(Enum): @@ -67,18 +67,18 @@ class Objective: name: str """name: The name of the objective. This should be something descriptive of the function or group of functions it covers.""" - success_rate: Union[None, ObjectivePercentile] + success_rate: Optional[ObjectivePercentile] """Specify the success rate for this objective. This means that the function or group of functions that are part of this objective should return an `Ok` result at least this percentage of the time.""" - latency = Union[None, Tuple[ObjectiveLatency, ObjectivePercentile]] + latency: Optional[Tuple[ObjectiveLatency, ObjectivePercentile]] def __init__( self, name: str, - success_rate: Union[None, ObjectivePercentile] = None, - latency: Union[None, Tuple[ObjectiveLatency, ObjectivePercentile]] = None, + success_rate: Optional[ObjectivePercentile] = None, + latency: Optional[Tuple[ObjectiveLatency, ObjectivePercentile]] = None, ): """Create a new objective with the given name. diff --git a/src/autometrics/prometheus_url.py b/src/autometrics/prometheus_url.py index 86c85ba..1f21f1a 100644 --- a/src/autometrics/prometheus_url.py +++ b/src/autometrics/prometheus_url.py @@ -1,23 +1,28 @@ import urllib.parse import os +from typing import Optional from dotenv import load_dotenv +def cleanup_url(url: str) -> str: + """Remove the trailing slash if there is one.""" + if url[-1] == "/": + url = url[:-1] + return url + + class Generator: """Generate prometheus query urls for a given function/module.""" - def __init__(self, function_name: str, module_name: str, base_url: str = None): + def __init__( + self, function_name: str, module_name: str, base_url: Optional[str] = None + ): load_dotenv() self.function_name = function_name self.module_name = module_name - self.base_url = base_url or os.getenv("PROMETHEUS_URL") - if self.base_url is None: - self.base_url = "http://localhost:9090" - elif self.base_url[-1] == "/": - self.base_url = self.base_url[ - :-1 - ] # Remove the trailing slash if there is one + url = base_url or os.getenv("PROMETHEUS_URL") or "http://localhost:9090" + self.base_url = cleanup_url(url) def create_urls(self): """Create the prometheus query urls for the function and module.""" @@ -39,5 +44,5 @@ def create_urls(self): def create_prometheus_url(self, query: str): """Create a the full query url for a given query.""" encoded_query = urllib.parse.quote(query) - url = f"{self.baseUrl}/graph?g0.expr={encoded_query}&g0.tab=0" + url = f"{self.base_url}/graph?g0.expr={encoded_query}&g0.tab=0" return url From 6734323535dbf54408144b2d309d2eb8a6d441b7 Mon Sep 17 00:00:00 2001 From: Jacco Flenter Date: Tue, 11 Apr 2023 11:17:46 +0200 Subject: [PATCH 11/23] Update test --- .github/workflows/build.yml | 7 +++- README.md | 27 +++++++++++++- poetry.lock | 74 +++++++++++++++++++++++++++++++++++-- pyproject.toml | 1 + src/test_prometheus_url.py | 10 +++-- 5 files changed, 107 insertions(+), 12 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0730264..32ad29b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -20,6 +20,9 @@ jobs: cache: "poetry" - name: Install dependencies run: poetry install --no-interaction --no-root --with dev - - uses: psf/black@stable - - name: Run pyright + - name: Check code formatting + uses: psf/black@stable + - name: Lint code run: poetry run pyright + - name: run test + run: poetry run pytest diff --git a/README.md b/README.md index 3178c1e..faf007a 100644 --- a/README.md +++ b/README.md @@ -55,6 +55,29 @@ This package uses [poetry](https://python-poetry.org) as a package manager, with By default, poetry will only install required dependencies, if you want to run examples, install using this command: -`poetry install --with examples` +```sh +poetry install --with examples +``` + +Code in this repository is: + +- formatted using [black](https://black.readthedocs.io/en/stable/). +- contains type definitions (which are linted by [pyright](https://microsoft.github.io/pyright/)) +- tested using [pytest](https://docs.pytest.org/) + +In order to run these tools locally you have to install them, you can install them using poetry: -Code in this repository is formatted using [black](https://black.readthedocs.io/en/stable/) and contains type definitions (which are linted by [pyright](https://microsoft.github.io/pyright/)) +```sh +poetry install --with dev +``` + +After that you can run the tools individually + +```sh +# Formatting using black +poetry run black . +# Lint using pyright +poetry run pyright +# Run the tests using pytest +poetry run pytest +``` diff --git a/poetry.lock b/poetry.lock index 55f171d..d7ca949 100644 --- a/poetry.lock +++ b/poetry.lock @@ -319,6 +319,21 @@ files = [ {file = "docutils-0.19.tar.gz", hash = "sha256:33995a6753c30b7f577febfc2c50411fec6aac7f7ffeb7c4cfe5991072dcf9e6"}, ] +[[package]] +name = "exceptiongroup" +version = "1.1.1" +description = "Backport of PEP 654 (exception groups)" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.1.1-py3-none-any.whl", hash = "sha256:232c37c63e4f682982c8b6459f33a8981039e5fb8756b2074364e5055c498c9e"}, + {file = "exceptiongroup-1.1.1.tar.gz", hash = "sha256:d484c3090ba2889ae2928419117447a14daf3c1231d5e30d0aae34f354f01785"}, +] + +[package.extras] +test = ["pytest (>=6)"] + [[package]] name = "fastapi" version = "0.95.0" @@ -404,6 +419,18 @@ zipp = {version = ">=3.1.0", markers = "python_version < \"3.10\""} docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] testing = ["flake8 (<5)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + [[package]] name = "jaraco-classes" version = "3.2.3" @@ -517,7 +544,7 @@ files = [ name = "nodeenv" version = "1.7.0" description = "Node.js virtual environment builder" -category = "main" +category = "dev" optional = false python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*" files = [ @@ -555,6 +582,22 @@ files = [ [package.extras] testing = ["pytest", "pytest-cov"] +[[package]] +name = "pluggy" +version = "1.0.0" +description = "plugin and hook calling mechanisms for python" +category = "dev" +optional = false +python-versions = ">=3.6" +files = [ + {file = "pluggy-1.0.0-py2.py3-none-any.whl", hash = "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"}, + {file = "pluggy-1.0.0.tar.gz", hash = "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + [[package]] name = "prometheus-client" version = "0.16.0" @@ -669,7 +712,7 @@ tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} name = "pyright" version = "1.1.302" description = "Command line wrapper for pyright" -category = "main" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -684,6 +727,29 @@ nodeenv = ">=1.6.0" all = ["twine (>=3.4.1)"] dev = ["twine (>=3.4.1)"] +[[package]] +name = "pytest" +version = "7.3.0" +description = "pytest: simple powerful testing with Python" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pytest-7.3.0-py3-none-any.whl", hash = "sha256:933051fa1bfbd38a21e73c3960cebdad4cf59483ddba7696c48509727e17f201"}, + {file = "pytest-7.3.0.tar.gz", hash = "sha256:58ecc27ebf0ea643ebfdf7fb1249335da761a00c9f955bcd922349bcb68ee57d"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "sys_platform == \"win32\""} +exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<2.0" +tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} + +[package.extras] +testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "xmlschema"] + [[package]] name = "python-dotenv" version = "1.0.0" @@ -823,7 +889,7 @@ jeepney = ">=0.6" name = "setuptools" version = "67.6.1" description = "Easily download, build, install, upgrade, and uninstall Python packages" -category = "main" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -993,4 +1059,4 @@ testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more [metadata] lock-version = "2.0" python-versions = "^3.8" -content-hash = "90aeb247a71d500a324409a2d7ae33e9d8150665f5851d8e0428f4905f146f20" +content-hash = "0387b1f4d7f8e51e6d5e58c7e6d5808d82e9cd33dc7d7ef4be7a44323581bf19" diff --git a/pyproject.toml b/pyproject.toml index c06785b..e7c5d38 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,6 +24,7 @@ optional = true [tool.poetry.group.dev.dependencies] pyright = "^1.1.302" +pytest = "^7.3.0" [tool.poetry.group.examples] optional = true diff --git a/src/test_prometheus_url.py b/src/test_prometheus_url.py index d06991e..dd45c40 100644 --- a/src/test_prometheus_url.py +++ b/src/test_prometheus_url.py @@ -4,10 +4,13 @@ # Defaults to localhost:9090 class TestPrometheusUrlGeneratorDefault(unittest.TestCase): + """Test the prometheus url generator with default values.""" + def setUp(self): self.generator = Generator("myFunction", "myModule") def test_create_prometheus_url(self): + """Test that the prometheus url is created correctly.""" url = self.generator.create_prometheus_url("myQuery") self.assertTrue( url.startswith("http://localhost:9090/graph?g0.expr=") @@ -17,18 +20,17 @@ def test_create_prometheus_url(self): # Creates proper urls when given a custom base URL class TestPrometheusUrlGeneratorCustomUrl(unittest.TestCase): + """Test the prometheus url generator with a custom base URL.""" + def setUp(self): self.generator = Generator( "myFunction", "myModule", base_url="http://localhost:9091" ) def test_create_prometheus_url(self): + """Test that the prometheus url is created correctly.""" url = self.generator.create_prometheus_url("myQuery") self.assertTrue( url.startswith("http://localhost:9091/graph?g0.expr=") ) # Make sure the base URL is correct self.assertIn("myQuery", url) # Make sure the query is included in the URL - - -if __name__ == "__main__": - unittest.main() From bcee99603ba21824da47c0ab027fa497e453ef65 Mon Sep 17 00:00:00 2001 From: Jacco Flenter Date: Tue, 11 Apr 2023 11:30:33 +0200 Subject: [PATCH 12/23] Move test code to fix build --- src/{ => autometrics}/test_prometheus_url.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename src/{ => autometrics}/test_prometheus_url.py (96%) diff --git a/src/test_prometheus_url.py b/src/autometrics/test_prometheus_url.py similarity index 96% rename from src/test_prometheus_url.py rename to src/autometrics/test_prometheus_url.py index dd45c40..a8c0c1d 100644 --- a/src/test_prometheus_url.py +++ b/src/autometrics/test_prometheus_url.py @@ -1,5 +1,5 @@ import unittest -from autometrics.prometheus_url import Generator +from .prometheus_url import Generator # Defaults to localhost:9090 From 4c5509970963f1412fc3ccb5b057b0a60468f3c3 Mon Sep 17 00:00:00 2001 From: Jacco Flenter Date: Tue, 11 Apr 2023 11:42:46 +0200 Subject: [PATCH 13/23] Fix missing typing_extensions --- poetry.lock | 4 ++-- pyproject.toml | 2 +- src/autometrics/decorator.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/poetry.lock b/poetry.lock index d7ca949..622a91d 100644 --- a/poetry.lock +++ b/poetry.lock @@ -984,7 +984,7 @@ urllib3 = ">=1.26.0" name = "typing-extensions" version = "4.5.0" description = "Backported and Experimental Type Hints for Python 3.7+" -category = "dev" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1059,4 +1059,4 @@ testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more [metadata] lock-version = "2.0" python-versions = "^3.8" -content-hash = "0387b1f4d7f8e51e6d5e58c7e6d5808d82e9cd33dc7d7ef4be7a44323581bf19" +content-hash = "aba99568a251557afb44dd53df1c14ef9a3992373790d702f9676c89b43d02a4" diff --git a/pyproject.toml b/pyproject.toml index e7c5d38..0e55208 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,6 +18,7 @@ packages = [{include = "autometrics", from = "src"}] python = "^3.8" prometheus-client = "0.16.0" python-dotenv = "1.0.0" +typing-extensions = "^4.5.0" [tool.poetry.group.dev] optional = true @@ -60,7 +61,6 @@ six = "1.16.0" sniffio = "1.3.0" starlette = "0.26.1" twine = "4.0.2" -typing-extensions = "4.5.0" urllib3 = "1.26.15" uvicorn = "0.21.1" webencodings = "0.5.1" diff --git a/src/autometrics/decorator.py b/src/autometrics/decorator.py index fca8e90..3fba103 100644 --- a/src/autometrics/decorator.py +++ b/src/autometrics/decorator.py @@ -1,8 +1,8 @@ """Autometrics module.""" import time -from typing import Union, overload, TypeVar, Any, Callable, Optional -from typing_extensions import ParamSpec from functools import wraps +from typing import overload, TypeVar, Callable, Optional +from typing_extensions import ParamSpec from .objectives import Objective from .emit import count, histogram, Result from .utils import get_module_name, get_caller_function, write_docs From 87a39e0348b5e2612d0519a59768c3506919451a Mon Sep 17 00:00:00 2001 From: Jacco Flenter Date: Tue, 11 Apr 2023 12:24:20 +0200 Subject: [PATCH 14/23] Update name of step in build action --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 32ad29b..795dac7 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -24,5 +24,5 @@ jobs: uses: psf/black@stable - name: Lint code run: poetry run pyright - - name: run test + - name: Run tests run: poetry run pytest From c9abcf29ec029898069d06bd145a6bdec813aed4 Mon Sep 17 00:00:00 2001 From: Jacco Flenter Date: Tue, 11 Apr 2023 23:12:37 +0200 Subject: [PATCH 15/23] Add tests for metrics --- src/autometrics/emit.py | 6 +- src/autometrics/objectives.py | 6 +- src/autometrics/test_decorator.py | 101 ++++++++++++++++++++++++++++++ src/autometrics/utils.py | 4 +- 4 files changed, 109 insertions(+), 8 deletions(-) create mode 100644 src/autometrics/test_decorator.py diff --git a/src/autometrics/emit.py b/src/autometrics/emit.py index 8dbe8d5..8d5de56 100644 --- a/src/autometrics/emit.py +++ b/src/autometrics/emit.py @@ -57,7 +57,7 @@ def count( percentile = ( "" if objective is None or objective.success_rate is None - else objective.success_rate + else objective.success_rate.value ) prom_counter.labels( @@ -84,8 +84,8 @@ def histogram( percentile = "" threshold = "" if latency is not None: - threshold = latency[0] - percentile = latency[1] + threshold = latency[0].value + percentile = latency[1].value prom_histogram.labels( func_name, diff --git a/src/autometrics/objectives.py b/src/autometrics/objectives.py index 7291cd0..bed1557 100644 --- a/src/autometrics/objectives.py +++ b/src/autometrics/objectives.py @@ -23,11 +23,11 @@ class ObjectiveLatency(Enum): Ms250 = "0.25" Ms500 = "0.5" Ms750 = "0.75" - Ms1000 = "1" + Ms1000 = "1.0" Ms2500 = "2.5" - Ms5000 = "5" + Ms5000 = "5.0" Ms7500 = "7.5" - Ms10000 = "10" + Ms10000 = "10.0" # This represents a Service-Level Objective (SLO) for a function or group of functions. diff --git a/src/autometrics/test_decorator.py b/src/autometrics/test_decorator.py new file mode 100644 index 0000000..cc2b098 --- /dev/null +++ b/src/autometrics/test_decorator.py @@ -0,0 +1,101 @@ +"""Test the autometrics decorator.""" +from prometheus_client.exposition import generate_latest +from prometheus_client import registry, Metric +from typing import Tuple + +from .decorator import autometrics +from .objectives import ObjectiveLatency, Objective, ObjectivePercentile +from .utils import get_caller_function + + +# @autometrics +def basic_function(): + """This is a basic function.""" + return True + + +def find_metric_with_name(metrics: "list[Metric]", name: str): + """Find a metric with a given name.""" + for metric in metrics: + if metric.name == name: + return metric + + return None + + +def test_basic(): + """This is a basic test.""" + + # set up the function + basic variables + caller = get_caller_function(depth=1) + assert caller is not None + assert caller != "" + function_name = basic_function.__name__ + wrapped_function = autometrics(basic_function) + wrapped_function() + + # get the metrics + blob = generate_latest(registry.REGISTRY) + assert blob is not None + data = blob.decode("utf-8") + + total_count = f"""function_calls_count_total{{caller="{caller}",function="{function_name}",module="test_decorator",objective_name="",objective_percentile="",result="Result.OK"}} 1.0""" + assert total_count in data + count_created = f"""function_calls_count_created{{caller="{caller}",function="{function_name}",module="test_decorator",objective_name="",objective_percentile="",result="Result.OK"}}""" + assert count_created in data + + for latency in ObjectiveLatency: + query = f"""function_calls_duration_bucket{{function="{function_name}",le="{latency.value}",module="test_decorator",objective_latency_threshold="",objective_name="",objective_percentile=""}}""" + assert query in data + + duration_count = f"""function_calls_duration_count{{function="{function_name}",module="test_decorator",objective_latency_threshold="",objective_name="",objective_percentile=""}}""" + assert duration_count in data + + duration_sum = f"""function_calls_duration_sum{{function="{function_name}",module="test_decorator",objective_latency_threshold="",objective_name="",objective_percentile=""}}""" + assert duration_sum in data + + duration_created = f"""function_calls_duration_created{{function="{function_name}",module="test_decorator",objective_latency_threshold="",objective_name="",objective_percentile=""}}""" + assert duration_created in data + + +def test_objectives(): + """This is a test that covers objectives.""" + + # set up the function + objective variables + caller = get_caller_function(depth=1) + assert caller is not None + assert caller != "" + objective_name = "test_objective" + success_rate = ObjectivePercentile.P90 + latency = (ObjectiveLatency.Ms100, ObjectivePercentile.P99) + objective = Objective( + name=objective_name, success_rate=success_rate, latency=latency + ) + function_name = basic_function.__name__ + wrapped_function = autometrics(objective=objective)(basic_function) + + # call the function + wrapped_function() + + # get the metrics + blob = generate_latest(registry.REGISTRY) + assert blob is not None + data = blob.decode("utf-8") + + total_count = f"""function_calls_count_total{{caller="{caller}",function="{function_name}",module="test_decorator",objective_name="{objective_name}",objective_percentile="{success_rate.value}",result="Result.OK"}} 1.0""" + assert total_count in data + count_created = f"""function_calls_count_created{{caller="{caller}",function="{function_name}",module="test_decorator",objective_name="{objective_name}",objective_percentile="{success_rate.value}",result="Result.OK"}}""" + assert count_created in data + + for objective in ObjectiveLatency: + query = f"""function_calls_duration_bucket{{function="{function_name}",le="{objective.value}",module="test_decorator",objective_latency_threshold="{latency[0].value}",objective_name="{objective_name}",objective_percentile="{latency[1].value}"}}""" + assert query in data + + duration_count = f"""function_calls_duration_count{{function="{function_name}",module="test_decorator",objective_latency_threshold="{latency[0].value}",objective_name="{objective_name}",objective_percentile="{latency[1].value}"}}""" + assert duration_count in data + + duration_sum = f"""function_calls_duration_sum{{function="{function_name}",module="test_decorator",objective_latency_threshold="{latency[0].value}",objective_name="{objective_name}",objective_percentile="{latency[1].value}"}}""" + assert duration_sum in data + + duration_created = f"""function_calls_duration_created{{function="{function_name}",module="test_decorator",objective_latency_threshold="{latency[0].value}",objective_name="{objective_name}",objective_percentile="{latency[1].value}"}}""" + assert duration_created in data diff --git a/src/autometrics/utils.py b/src/autometrics/utils.py index dc6258c..10fe144 100644 --- a/src/autometrics/utils.py +++ b/src/autometrics/utils.py @@ -40,8 +40,8 @@ def write_docs(func_name: str, module_name: str): return docs -def get_caller_function(): +def get_caller_function(depth: int = 2): """Get the name of the function that called the function being decorated.""" - caller_frame = inspect.stack()[2] + caller_frame = inspect.stack()[depth] caller_function_name = caller_frame[3] return caller_function_name From 7796cc5332538097bfaf2dd9c2779ae8f41c688f Mon Sep 17 00:00:00 2001 From: Jacco Flenter Date: Wed, 12 Apr 2023 09:52:03 +0200 Subject: [PATCH 16/23] Add black to the dev dependencies And use that in CI --- .github/workflows/build.yml | 2 +- poetry.lock | 92 ++++++++++++++++++++++++++++++++++++- pyproject.toml | 1 + 3 files changed, 93 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 795dac7..8ba8db4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -21,7 +21,7 @@ jobs: - name: Install dependencies run: poetry install --no-interaction --no-root --with dev - name: Check code formatting - uses: psf/black@stable + run: poetry run black . - name: Lint code run: poetry run pyright - name: Run tests diff --git a/poetry.lock b/poetry.lock index 622a91d..c04093e 100644 --- a/poetry.lock +++ b/poetry.lock @@ -21,6 +21,56 @@ doc = ["packaging", "sphinx-autodoc-typehints (>=1.2.0)", "sphinx-rtd-theme"] test = ["contextlib2", "coverage[toml] (>=4.5)", "hypothesis (>=4.0)", "mock (>=4)", "pytest (>=7.0)", "pytest-mock (>=3.6.1)", "trustme", "uvloop (<0.15)", "uvloop (>=0.15)"] trio = ["trio (>=0.16,<0.22)"] +[[package]] +name = "black" +version = "23.3.0" +description = "The uncompromising code formatter." +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "black-23.3.0-cp310-cp310-macosx_10_16_arm64.whl", hash = "sha256:0945e13506be58bf7db93ee5853243eb368ace1c08a24c65ce108986eac65915"}, + {file = "black-23.3.0-cp310-cp310-macosx_10_16_universal2.whl", hash = "sha256:67de8d0c209eb5b330cce2469503de11bca4085880d62f1628bd9972cc3366b9"}, + {file = "black-23.3.0-cp310-cp310-macosx_10_16_x86_64.whl", hash = "sha256:7c3eb7cea23904399866c55826b31c1f55bbcd3890ce22ff70466b907b6775c2"}, + {file = "black-23.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:32daa9783106c28815d05b724238e30718f34155653d4d6e125dc7daec8e260c"}, + {file = "black-23.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:35d1381d7a22cc5b2be2f72c7dfdae4072a3336060635718cc7e1ede24221d6c"}, + {file = "black-23.3.0-cp311-cp311-macosx_10_16_arm64.whl", hash = "sha256:a8a968125d0a6a404842fa1bf0b349a568634f856aa08ffaff40ae0dfa52e7c6"}, + {file = "black-23.3.0-cp311-cp311-macosx_10_16_universal2.whl", hash = "sha256:c7ab5790333c448903c4b721b59c0d80b11fe5e9803d8703e84dcb8da56fec1b"}, + {file = "black-23.3.0-cp311-cp311-macosx_10_16_x86_64.whl", hash = "sha256:a6f6886c9869d4daae2d1715ce34a19bbc4b95006d20ed785ca00fa03cba312d"}, + {file = "black-23.3.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6f3c333ea1dd6771b2d3777482429864f8e258899f6ff05826c3a4fcc5ce3f70"}, + {file = "black-23.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:11c410f71b876f961d1de77b9699ad19f939094c3a677323f43d7a29855fe326"}, + {file = "black-23.3.0-cp37-cp37m-macosx_10_16_x86_64.whl", hash = "sha256:1d06691f1eb8de91cd1b322f21e3bfc9efe0c7ca1f0e1eb1db44ea367dff656b"}, + {file = "black-23.3.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:50cb33cac881766a5cd9913e10ff75b1e8eb71babf4c7104f2e9c52da1fb7de2"}, + {file = "black-23.3.0-cp37-cp37m-win_amd64.whl", hash = "sha256:e114420bf26b90d4b9daa597351337762b63039752bdf72bf361364c1aa05925"}, + {file = "black-23.3.0-cp38-cp38-macosx_10_16_arm64.whl", hash = "sha256:48f9d345675bb7fbc3dd85821b12487e1b9a75242028adad0333ce36ed2a6d27"}, + {file = "black-23.3.0-cp38-cp38-macosx_10_16_universal2.whl", hash = "sha256:714290490c18fb0126baa0fca0a54ee795f7502b44177e1ce7624ba1c00f2331"}, + {file = "black-23.3.0-cp38-cp38-macosx_10_16_x86_64.whl", hash = "sha256:064101748afa12ad2291c2b91c960be28b817c0c7eaa35bec09cc63aa56493c5"}, + {file = "black-23.3.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:562bd3a70495facf56814293149e51aa1be9931567474993c7942ff7d3533961"}, + {file = "black-23.3.0-cp38-cp38-win_amd64.whl", hash = "sha256:e198cf27888ad6f4ff331ca1c48ffc038848ea9f031a3b40ba36aced7e22f2c8"}, + {file = "black-23.3.0-cp39-cp39-macosx_10_16_arm64.whl", hash = "sha256:3238f2aacf827d18d26db07524e44741233ae09a584273aa059066d644ca7b30"}, + {file = "black-23.3.0-cp39-cp39-macosx_10_16_universal2.whl", hash = "sha256:f0bd2f4a58d6666500542b26354978218a9babcdc972722f4bf90779524515f3"}, + {file = "black-23.3.0-cp39-cp39-macosx_10_16_x86_64.whl", hash = "sha256:92c543f6854c28a3c7f39f4d9b7694f9a6eb9d3c5e2ece488c327b6e7ea9b266"}, + {file = "black-23.3.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a150542a204124ed00683f0db1f5cf1c2aaaa9cc3495b7a3b5976fb136090ab"}, + {file = "black-23.3.0-cp39-cp39-win_amd64.whl", hash = "sha256:6b39abdfb402002b8a7d030ccc85cf5afff64ee90fa4c5aebc531e3ad0175ddb"}, + {file = "black-23.3.0-py3-none-any.whl", hash = "sha256:ec751418022185b0c1bb7d7736e6933d40bbb14c14a0abcf9123d1b159f98dd4"}, + {file = "black-23.3.0.tar.gz", hash = "sha256:1c7b8d606e728a41ea1ccbd7264677e494e87cf630e399262ced92d4a8dac940"}, +] + +[package.dependencies] +click = ">=8.0.0" +mypy-extensions = ">=0.4.3" +packaging = ">=22.0" +pathspec = ">=0.9.0" +platformdirs = ">=2" +tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} +typing-extensions = {version = ">=3.10.0.0", markers = "python_version < \"3.10\""} + +[package.extras] +colorama = ["colorama (>=0.4.3)"] +d = ["aiohttp (>=3.7.4)"] +jupyter = ["ipython (>=7.8.0)", "tokenize-rt (>=3.2.0)"] +uvloop = ["uvloop (>=0.15.2)"] + [[package]] name = "bleach" version = "6.0.0" @@ -540,6 +590,18 @@ files = [ {file = "more_itertools-9.1.0-py3-none-any.whl", hash = "sha256:d2bc7f02446e86a68911e58ded76d6561eea00cddfb2a91e7019bbb586c799f3"}, ] +[[package]] +name = "mypy-extensions" +version = "1.0.0" +description = "Type system extensions for programs checked with the mypy type checker." +category = "dev" +optional = false +python-versions = ">=3.5" +files = [ + {file = "mypy_extensions-1.0.0-py3-none-any.whl", hash = "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d"}, + {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"}, +] + [[package]] name = "nodeenv" version = "1.7.0" @@ -567,6 +629,18 @@ files = [ {file = "packaging-23.0.tar.gz", hash = "sha256:b6ad297f8907de0fa2fe1ccbd26fdaf387f5f47c7275fedf8cce89f99446cf97"}, ] +[[package]] +name = "pathspec" +version = "0.11.1" +description = "Utility library for gitignore style pattern matching of file paths." +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pathspec-0.11.1-py3-none-any.whl", hash = "sha256:d8af70af76652554bd134c22b3e8a1cc46ed7d91edcdd721ef1a0c51a84a5293"}, + {file = "pathspec-0.11.1.tar.gz", hash = "sha256:2798de800fa92780e33acca925945e9a19a133b715067cf165b8866c15a31687"}, +] + [[package]] name = "pkginfo" version = "1.9.6" @@ -582,6 +656,22 @@ files = [ [package.extras] testing = ["pytest", "pytest-cov"] +[[package]] +name = "platformdirs" +version = "3.2.0" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "platformdirs-3.2.0-py3-none-any.whl", hash = "sha256:ebe11c0d7a805086e99506aa331612429a72ca7cd52a1f0d277dc4adc20cb10e"}, + {file = "platformdirs-3.2.0.tar.gz", hash = "sha256:d5b638ca397f25f979350ff789db335903d7ea010ab28903f57b27e1b16c2b08"}, +] + +[package.extras] +docs = ["furo (>=2022.12.7)", "proselint (>=0.13)", "sphinx (>=6.1.3)", "sphinx-autodoc-typehints (>=1.22,!=1.23.4)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.2.2)", "pytest-cov (>=4)", "pytest-mock (>=3.10)"] + [[package]] name = "pluggy" version = "1.0.0" @@ -1059,4 +1149,4 @@ testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more [metadata] lock-version = "2.0" python-versions = "^3.8" -content-hash = "aba99568a251557afb44dd53df1c14ef9a3992373790d702f9676c89b43d02a4" +content-hash = "ac28fcba5ed105020dd055b90b69c98b3f0efa34f8f3039efbb3c0015e89c88d" diff --git a/pyproject.toml b/pyproject.toml index 0e55208..5dbdeef 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,6 +26,7 @@ optional = true [tool.poetry.group.dev.dependencies] pyright = "^1.1.302" pytest = "^7.3.0" +black = "^23.3.0" [tool.poetry.group.examples] optional = true From f93c28b40ffe0590f92ebd314599fe55e1e55c45 Mon Sep 17 00:00:00 2001 From: Jacco Flenter Date: Wed, 12 Apr 2023 10:49:16 +0200 Subject: [PATCH 17/23] Update readme --- README.md | 31 ++++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index faf007a..d2ec02d 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,8 @@ +![GitHub_headerImage](https://user-images.githubusercontent.com/3262610/221191767-73b8a8d9-9f8b-440e-8ab6-75cb3c82f2bc.png) + # autometrics-py -A Python decorator that makes it easy to understand the error rate, response time, and production usage of any function in your code. Jump straight from your IDE to live Prometheus charts for each HTTP/RPC handler, database method, or other piece of application logic. +A Python library that exports a decorator that makes it easy to understand the error rate, response time, and production usage of any function in your code. Jump straight from your IDE to live Prometheus charts for each HTTP/RPC handler, database method, or other piece of application logic. Autometrics for Python provides: @@ -18,8 +20,7 @@ See [Why Autometrics?](https://github.com/autometrics-dev#why-autometrics) for m - 🔗 Create links to live Prometheus charts directly into each functions docstrings (with tooltips coming soon!) - 📊 (Coming Soon!) Grafana dashboard showing the performance of all instrumented functions -- 🚨 Emits data for Prometheus alerting rules using SLO best practices - from simple annotations in your code +- 🚨 Enable Prometheus alerts using SLO best practices from simple annotations in your code - ⚡ Minimal runtime overhead ## Using autometrics-py @@ -45,6 +46,30 @@ def sayHello: > Note that we cannot support tooltips without a VSCode extension due to behavior of the [static analyzer](https://github.com/davidhalter/jedi/issues/1921) used in VSCode. +## Alerts / SLOs + +Autometrics makes it easy to add Prometheus alerts using Service-Level Objectives (SLOs) to a function or group of functions. + +This works using pre-defined [Prometheus alerting rules](./autometrics.rules.yml) (read more about alerting rules in general [here](https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/)). +By default, most of the recording rules are dormaint. They are enabled by specific metric labels that can be automatically attached by autometrics. + +To use autometrics SLOs and alerts, create one or multiple `Objective`s based on the function(s) success rate and/or latency, as shown below. The `Objective` can be passed as an argument to the `autometrics` macro to include the given function in that objective. + +```python +from autometrics import autometrics +from autometrics.objectives import Objective, ObjectiveLatency, ObjectivePercentile + +API_SLO = Objective( + "random", + success_rate=ObjectivePercentile.P99_9, + latency=(ObjectiveLatency.Ms250, ObjectivePercentile.P99), +) + +@autometrics(objective=API_SLO) +def api_handler(): + # ... +``` + ## Development of the package This package uses [poetry](https://python-poetry.org) as a package manager, with all dependencies separated into three groups: From ec40460abfaa404b29a73f7db12cf175cd600ed4 Mon Sep 17 00:00:00 2001 From: Jacco Flenter Date: Wed, 12 Apr 2023 11:03:24 +0200 Subject: [PATCH 18/23] Tweak example error function --- examples/example.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/example.py b/examples/example.py index bcbd657..f60cbea 100644 --- a/examples/example.py +++ b/examples/example.py @@ -46,12 +46,12 @@ def div_unhandled(num1, num2): @autometrics(objective=RANDOM_SLO) def random_error(): - """This function will randomly return an error or not.""" + """This function will randomly return an error or ok.""" result = random.choice(["ok", "error"]) if result == "error": time.sleep(1) - raise Exception("random error") + raise RuntimeError("random error") return result From 1869f79d679e7d4823dfc23a40d4d0ff042c1428 Mon Sep 17 00:00:00 2001 From: Jacco Flenter Date: Wed, 12 Apr 2023 11:35:27 +0200 Subject: [PATCH 19/23] Add test for exceptions --- src/autometrics/emit.py | 2 +- src/autometrics/test_decorator.py | 51 +++++++++++++++++++++++++++---- 2 files changed, 46 insertions(+), 7 deletions(-) diff --git a/src/autometrics/emit.py b/src/autometrics/emit.py index 8d5de56..f16e240 100644 --- a/src/autometrics/emit.py +++ b/src/autometrics/emit.py @@ -63,7 +63,7 @@ def count( prom_counter.labels( func_name, module_name, - result, + result.value, caller, objective_name, percentile, diff --git a/src/autometrics/test_decorator.py b/src/autometrics/test_decorator.py index cc2b098..c4c767f 100644 --- a/src/autometrics/test_decorator.py +++ b/src/autometrics/test_decorator.py @@ -1,14 +1,13 @@ """Test the autometrics decorator.""" from prometheus_client.exposition import generate_latest from prometheus_client import registry, Metric -from typing import Tuple +from pytest import raises from .decorator import autometrics from .objectives import ObjectiveLatency, Objective, ObjectivePercentile from .utils import get_caller_function -# @autometrics def basic_function(): """This is a basic function.""" return True @@ -39,9 +38,9 @@ def test_basic(): assert blob is not None data = blob.decode("utf-8") - total_count = f"""function_calls_count_total{{caller="{caller}",function="{function_name}",module="test_decorator",objective_name="",objective_percentile="",result="Result.OK"}} 1.0""" + total_count = f"""function_calls_count_total{{caller="{caller}",function="{function_name}",module="test_decorator",objective_name="",objective_percentile="",result="ok"}} 1.0""" assert total_count in data - count_created = f"""function_calls_count_created{{caller="{caller}",function="{function_name}",module="test_decorator",objective_name="",objective_percentile="",result="Result.OK"}}""" + count_created = f"""function_calls_count_created{{caller="{caller}",function="{function_name}",module="test_decorator",objective_name="",objective_percentile="",result="ok"}}""" assert count_created in data for latency in ObjectiveLatency: @@ -82,9 +81,9 @@ def test_objectives(): assert blob is not None data = blob.decode("utf-8") - total_count = f"""function_calls_count_total{{caller="{caller}",function="{function_name}",module="test_decorator",objective_name="{objective_name}",objective_percentile="{success_rate.value}",result="Result.OK"}} 1.0""" + total_count = f"""function_calls_count_total{{caller="{caller}",function="{function_name}",module="test_decorator",objective_name="{objective_name}",objective_percentile="{success_rate.value}",result="ok"}} 1.0""" assert total_count in data - count_created = f"""function_calls_count_created{{caller="{caller}",function="{function_name}",module="test_decorator",objective_name="{objective_name}",objective_percentile="{success_rate.value}",result="Result.OK"}}""" + count_created = f"""function_calls_count_created{{caller="{caller}",function="{function_name}",module="test_decorator",objective_name="{objective_name}",objective_percentile="{success_rate.value}",result="ok"}}""" assert count_created in data for objective in ObjectiveLatency: @@ -99,3 +98,43 @@ def test_objectives(): duration_created = f"""function_calls_duration_created{{function="{function_name}",module="test_decorator",objective_latency_threshold="{latency[0].value}",objective_name="{objective_name}",objective_percentile="{latency[1].value}"}}""" assert duration_created in data + + +def error_function(): + """This is a function that raises an error.""" + raise RuntimeError("This is a test error") + + +def test_exception(): + caller = get_caller_function(depth=1) + assert caller is not None + assert caller != "" + function_name = error_function.__name__ + wrapped_function = autometrics(error_function) + with raises(RuntimeError) as exception: + wrapped_function() + assert "This is a test error" in str(exception.value) + + # get the metrics + blob = generate_latest(registry.REGISTRY) + assert blob is not None + data = blob.decode("utf-8") + print("data", data) + + total_count = f"""function_calls_count_total{{caller="{caller}",function="{function_name}",module="test_decorator",objective_name="",objective_percentile="",result="error"}} 1.0""" + assert total_count in data + count_created = f"""function_calls_count_created{{caller="{caller}",function="{function_name}",module="test_decorator",objective_name="",objective_percentile="",result="error"}}""" + assert count_created in data + + for latency in ObjectiveLatency: + query = f"""function_calls_duration_bucket{{function="{function_name}",le="{latency.value}",module="test_decorator",objective_latency_threshold="",objective_name="",objective_percentile=""}}""" + assert query in data + + duration_count = f"""function_calls_duration_count{{function="{function_name}",module="test_decorator",objective_latency_threshold="",objective_name="",objective_percentile=""}}""" + assert duration_count in data + + duration_sum = f"""function_calls_duration_sum{{function="{function_name}",module="test_decorator",objective_latency_threshold="",objective_name="",objective_percentile=""}}""" + assert duration_sum in data + + duration_created = f"""function_calls_duration_created{{function="{function_name}",module="test_decorator",objective_latency_threshold="",objective_name="",objective_percentile=""}}""" + assert duration_created in data From 4e4fc2c59106af788790415b22dfad36c5ee95cc Mon Sep 17 00:00:00 2001 From: Jacco Flenter Date: Wed, 12 Apr 2023 17:34:19 +0200 Subject: [PATCH 20/23] Add information about the order of decorators --- README.md | 32 ++++++++++++++++++++++++++++++-- src/autometrics/utils.py | 2 +- 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index d2ec02d..c873f18 100644 --- a/README.md +++ b/README.md @@ -50,8 +50,7 @@ def sayHello: Autometrics makes it easy to add Prometheus alerts using Service-Level Objectives (SLOs) to a function or group of functions. -This works using pre-defined [Prometheus alerting rules](./autometrics.rules.yml) (read more about alerting rules in general [here](https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/)). -By default, most of the recording rules are dormaint. They are enabled by specific metric labels that can be automatically attached by autometrics. +This works using pre-defined [Prometheus alerting rules](https://github.com/autometrics-dev/autometrics-shared#prometheus-recording--alerting-rules). By default, most of the recording rules are dormant. They are enabled by specific metric labels that can be automatically attached by autometrics. To use autometrics SLOs and alerts, create one or multiple `Objective`s based on the function(s) success rate and/or latency, as shown below. The `Objective` can be passed as an argument to the `autometrics` macro to include the given function in that objective. @@ -70,6 +69,35 @@ def api_handler(): # ... ``` +Autometrics by default will try to store information on which function calls a decorated function. As such you may want to place the autometrics is the top/first decorator as otherwise you may get `inner` or `wrapper` as the caller function. + +So instead of writing: + +```py +def noop(func: Callable[..., R]) -> Callable[..., R]: + """A noop decorator that does nothing.""" + + @wraps(func) + def inner(*args: Any, **kwargs: Any) -> Any: + return func(*args, **kwargs) + + return inner + +@noop +@autometrics +def api_handler(): + # ... +``` + +You may want to: + +```py +@autometrics +@noop +def api_handler(): + # ... +``` + ## Development of the package This package uses [poetry](https://python-poetry.org) as a package manager, with all dependencies separated into three groups: diff --git a/src/autometrics/utils.py b/src/autometrics/utils.py index 10fe144..1dd9cf4 100644 --- a/src/autometrics/utils.py +++ b/src/autometrics/utils.py @@ -41,7 +41,7 @@ def write_docs(func_name: str, module_name: str): def get_caller_function(depth: int = 2): - """Get the name of the function that called the function being decorated.""" + """Get the name of the function. Default depth is 2 to get the caller of the caller of the function being decorated.""" caller_frame = inspect.stack()[depth] caller_function_name = caller_frame[3] return caller_function_name From a0c911aeed66d5c10c0b2d1921dcaaa99ddbae6b Mon Sep 17 00:00:00 2001 From: Brett Beutell Date: Wed, 12 Apr 2023 20:26:07 +0300 Subject: [PATCH 21/23] Remove dangling close bracket in doc string --- src/autometrics/objectives.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/autometrics/objectives.py b/src/autometrics/objectives.py index bed1557..73e89ca 100644 --- a/src/autometrics/objectives.py +++ b/src/autometrics/objectives.py @@ -46,7 +46,7 @@ class ObjectiveLatency(Enum): # latency=(ObjectiveLatency.Ms250, ObjectivePercentile.P99), # ) # -# @autometrics(objective = API_SLO)] +# @autometrics(objective = API_SLO) # def api_handler() : # # ... # ``` From 07ef7395f062f7ed97f1e4cf63a98663059ae80f Mon Sep 17 00:00:00 2001 From: Brett Beutell Date: Wed, 12 Apr 2023 20:29:15 +0300 Subject: [PATCH 22/23] Fix small typo in README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c873f18..2622b29 100644 --- a/README.md +++ b/README.md @@ -69,7 +69,7 @@ def api_handler(): # ... ``` -Autometrics by default will try to store information on which function calls a decorated function. As such you may want to place the autometrics is the top/first decorator as otherwise you may get `inner` or `wrapper` as the caller function. +Autometrics by default will try to store information on which function calls a decorated function. As such you may want to place the autometrics in the top/first decorator, as otherwise you may get `inner` or `wrapper` as the caller function. So instead of writing: From 0d8b65adacfabc789d48b80c5ba4410768b52ecf Mon Sep 17 00:00:00 2001 From: Jacco Flenter Date: Thu, 13 Apr 2023 11:58:00 +0200 Subject: [PATCH 23/23] Update readme --- README.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 2622b29..ed8a01c 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,7 @@ def sayHello: Autometrics makes it easy to add Prometheus alerts using Service-Level Objectives (SLOs) to a function or group of functions. -This works using pre-defined [Prometheus alerting rules](https://github.com/autometrics-dev/autometrics-shared#prometheus-recording--alerting-rules). By default, most of the recording rules are dormant. They are enabled by specific metric labels that can be automatically attached by autometrics. +In order to receive alerts you need to add a set of rules to your Prometheus set up. You can find out more about those rules here: [Prometheus alerting rules](https://github.com/autometrics-dev/autometrics-shared#prometheus-recording--alerting-rules). Once added, most of the recording rules are dormant. They are enabled by specific metric labels that can be automatically attached by autometrics. To use autometrics SLOs and alerts, create one or multiple `Objective`s based on the function(s) success rate and/or latency, as shown below. The `Objective` can be passed as an argument to the `autometrics` macro to include the given function in that objective. @@ -74,6 +74,11 @@ Autometrics by default will try to store information on which function calls a d So instead of writing: ```py +from functools import wraps +from typing import Any, TypeVar, Callable + +R = TypeVar("R") + def noop(func: Callable[..., R]) -> Callable[..., R]: """A noop decorator that does nothing.""" @@ -89,7 +94,7 @@ def api_handler(): # ... ``` -You may want to: +You may want to switch the order of the decorator ```py @autometrics