From d70793ac359ac152a64e1a7d5b37c0242e3cd7cd Mon Sep 17 00:00:00 2001 From: Diego Hurtado Date: Mon, 11 Dec 2023 18:11:54 -0600 Subject: [PATCH] Add support for environment variable substitution Fixes #60 --- .../opentelemetry/configuration/__init__.py | 6 +- .../configuration/_internal/__init__.py | 114 +++++- ...kitchen-sink.yaml => configuration_0.yaml} | 0 .../python/tests/data/configuration_1.yaml | 380 ++++++++++++++++++ prototypes/python/tests/test_configuration.py | 53 ++- 5 files changed, 532 insertions(+), 21 deletions(-) rename prototypes/python/tests/data/{kitchen-sink.yaml => configuration_0.yaml} (100%) create mode 100644 prototypes/python/tests/data/configuration_1.yaml diff --git a/prototypes/python/src/opentelemetry/configuration/__init__.py b/prototypes/python/src/opentelemetry/configuration/__init__.py index afe051a..e2dbadc 100644 --- a/prototypes/python/src/opentelemetry/configuration/__init__.py +++ b/prototypes/python/src/opentelemetry/configuration/__init__.py @@ -24,7 +24,8 @@ process_schema, render_schema, create_object, - load_configuration + load_configuration, + substitute_environment_variables, ) __all__ = [ @@ -33,5 +34,6 @@ "process_schema", "render_schema", "create_object", - "load_configuration" + "load_configuration", + "substitute_environment_variables", ] diff --git a/prototypes/python/src/opentelemetry/configuration/_internal/__init__.py b/prototypes/python/src/opentelemetry/configuration/_internal/__init__.py index e93df4f..b8903cb 100644 --- a/prototypes/python/src/opentelemetry/configuration/_internal/__init__.py +++ b/prototypes/python/src/opentelemetry/configuration/_internal/__init__.py @@ -12,7 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +from ipdb import set_trace +from os import environ from yaml import safe_load +from re import compile as re_compile from jsonref import JsonRef from os.path import exists from pathlib import Path @@ -25,6 +28,18 @@ from opentelemetry.configuration._internal.path_function import path_function from jinja2 import Environment, FileSystemLoader +set_trace + +_environment_variable_regex = re_compile(r"\$\{([a-zA-Z]\w*)\}") +_type_type = { + "integer": int, + "boolean": bool, + "string": str, + "array": list, + "object": object, + "number": float +} + def resolve_schema(json_file_path) -> dict: @@ -69,15 +84,6 @@ def retrieve_from_path(path: str): def process_schema(schema: dict) -> dict: - type_type = { - "integer": "int", - "boolean": "bool", - "string": "str", - "array": "list", - "object": "object", - "number": "float" - } - def traverse( schema: dict, schema_value_id_stack: list, @@ -117,21 +123,28 @@ def traverse( for positional_attribute in positional_attributes: result_positional_attributes[positional_attribute] = ( - type_type[ - schema_properties[positional_attribute]["type"] - ] + str( + _type_type[ + schema_properties[positional_attribute]["type"] + ].__name__ + ) ) for optional_attribute in optional_attributes: result_optional_attributes[optional_attribute] = ( - type_type[ - schema_properties[optional_attribute]["type"] - ] + str( + _type_type[ + schema_properties[optional_attribute]["type"] + ].__name__ + ) ) children = {} + children.update(result_positional_attributes) + children.update(result_optional_attributes) + processed_schema[schema_key_stack[-1]] = { "function_name": "_".join(schema_key_stack[1:]), "positional_attributes": result_positional_attributes, @@ -213,7 +226,7 @@ def traverse( return processed_schema[""]["children"] -def render_schema(processed_schema: dict): +def render_schema(processed_schema: dict, path_function_path: Path): def traverse( processed_schema: dict, @@ -226,6 +239,9 @@ def traverse( processed_schema_value ) in processed_schema.items(): + if not isinstance(processed_schema_value, dict): + continue + function_arguments[processed_schema_value["function_name"]] = { "optional_attributes": ( processed_schema_value["optional_attributes"] @@ -262,7 +278,7 @@ def traverse( loader=FileSystemLoader(current_path.joinpath("templates")) ) - with open("path_function.py", "w") as result_py_file: + with open(path_function_path, "w") as result_py_file: result_py_file.write( "\n".join( @@ -370,3 +386,67 @@ def create_object( processed_schema, path_function, ) + + +def substitute_environment_variables( + configuration: dict, + processed_schema: dict +) -> dict: + + def traverse( + configuration: dict, + processed_schema: dict, + original_processed_schema: dict + ): + + for configuration_key, configuration_value in configuration.items(): + + if configuration_key not in processed_schema.keys(): + continue + + if isinstance(configuration_value, dict): + + recursive_paths = ( + processed_schema[configuration_key]["recursive_path"] + ) + + if recursive_paths: + + children = original_processed_schema + + for recursive_path in recursive_paths: + children = children[recursive_path]["children"] + + else: + children = processed_schema[configuration_key]["children"] + + traverse( + configuration_value, + children, + original_processed_schema + ) + + elif isinstance(configuration_value, list): + + for element in configuration_value: + if isinstance(element, dict): + traverse( + element, + processed_schema[configuration_key]["children"], + original_processed_schema + ) + + elif isinstance(configuration_value, str): + + match = _environment_variable_regex.match(configuration_value) + + if match is not None: + + configuration[configuration_key] = ( + __builtins__[processed_schema[configuration_key]] + (environ.get(match.group(1))) + ) + + traverse(configuration, processed_schema, processed_schema) + + return configuration diff --git a/prototypes/python/tests/data/kitchen-sink.yaml b/prototypes/python/tests/data/configuration_0.yaml similarity index 100% rename from prototypes/python/tests/data/kitchen-sink.yaml rename to prototypes/python/tests/data/configuration_0.yaml diff --git a/prototypes/python/tests/data/configuration_1.yaml b/prototypes/python/tests/data/configuration_1.yaml new file mode 100644 index 0000000..83f4d97 --- /dev/null +++ b/prototypes/python/tests/data/configuration_1.yaml @@ -0,0 +1,380 @@ +# kitchen-sink.yaml demonstrates all configurable surface area, including explanatory comments. +# +# It DOES NOT represent expected real world configuration, as it makes strange configuration +# choices in an effort to exercise the full surface area. +# +# Configuration values are set to their defaults when default values are defined. + +# The file format version +file_format: "0.1" + +# Configure if the SDK is disabled or not. This is not required to be provided +# to ensure the SDK isn't disabled, the default value when this is not provided +# is for the SDK to be enabled. +# +# Environment variable: OTEL_SDK_DISABLED +disabled: false + +# Configure general attribute limits. See also tracer_provider.limits, logger_provider.limits. +attribute_limits: + # Configure max attribute value size. + # + # Environment variable: OTEL_ATTRIBUTE_VALUE_LENGTH_LIMIT + attribute_value_length_limit: 4096 + # Configure max attribute count. + # + # Environment variable: OTEL_ATTRIBUTE_COUNT_LIMIT + attribute_count_limit: 128 + +# Configure logger provider. +logger_provider: + # Configure log record processors. + processors: + # Configure a batch log record processor. + - batch: + # Configure delay interval (in milliseconds) between two consecutive exports. + # + # Environment variable: OTEL_BLRP_SCHEDULE_DELAY + schedule_delay: 5000 + # Configure maximum allowed time (in milliseconds) to export data. + # + # Environment variable: OTEL_BLRP_EXPORT_TIMEOUT + export_timeout: ${OTEL_BLRB_EXPORT_TIMEOUT} + # Configure maximum queue size. + # + # Environment variable: OTEL_BLRP_MAX_QUEUE_SIZE + max_queue_size: 2048 + # Configure maximum batch size. + # + # Environment variable: OTEL_BLRP_MAX_EXPORT_BATCH_SIZE + max_export_batch_size: 512 + # Configure exporter. + # + # Environment variable: OTEL_LOGS_EXPORTER + exporter: + # Configure exporter to be OTLP. + otlp: + # Configure protocol. + # + # Environment variable: OTEL_EXPORTER_OTLP_PROTOCOL, OTEL_EXPORTER_OTLP_LOGS_PROTOCOL + protocol: http/protobuf + # Configure endpoint. + # + # Environment variable: OTEL_EXPORTER_OTLP_ENDPOINT, OTEL_EXPORTER_OTLP_LOGS_ENDPOINT + endpoint: http://localhost:4318 + # Configure certificate. + # + # Environment variable: OTEL_EXPORTER_OTLP_CERTIFICATE, OTEL_EXPORTER_OTLP_LOGS_CERTIFICATE + certificate: /app/cert.pem + # Configure mTLS private client key. + # + # Environment variable: OTEL_EXPORTER_OTLP_CLIENT_KEY, OTEL_EXPORTER_OTLP_LOGS_CLIENT_KEY + client_key: /app/cert.pem + # Configure mTLS client certificate. + # + # Environment variable: OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE, OTEL_EXPORTER_OTLP_LOGS_CLIENT_CERTIFICATE + client_certificate: /app/cert.pem + # Configure headers. + # + # Environment variable: OTEL_EXPORTER_OTLP_HEADERS, OTEL_EXPORTER_OTLP_LOGS_HEADERS + headers: + api-key: "1234" + # Configure compression. + # + # Environment variable: OTEL_EXPORTER_OTLP_COMPRESSION, OTEL_EXPORTER_OTLP_LOGS_COMPRESSION + compression: gzip + # Configure max time (in milliseconds) to wait for each export. + # + # Environment variable: OTEL_EXPORTER_OTLP_TIMEOUT, OTEL_EXPORTER_OTLP_LOGS_TIMEOUT + timeout: 10000 + # Configure log record limits. See also attribute_limits. + limits: + # Configure max log record attribute value size. Overrides attribute_limits.attribute_value_length_limit. + # + # Environment variable: OTEL_LOGRECORD_ATTRIBUTE_VALUE_LENGTH_LIMIT + attribute_value_length_limit: 4096 + # Configure max log record attribute count. Overrides attribute_limits.attribute_count_limit. + # + # Environment variable: OTEL_LOGRECORD_ATTRIBUTE_COUNT_LIMIT + attribute_count_limit: 128 + +# Configure meter provider. +meter_provider: + # Configure metric readers. + readers: + # Configure a pull-based metric reader. + - pull: + # Configure exporter. + # + # Environment variable: OTEL_METRICS_EXPORTER + exporter: + # Configure exporter to be prometheus. + prometheus: + # Configure host. + # + # Environment variable: OTEL_EXPORTER_PROMETHEUS_HOST + host: localhost + # Configure port. + # + # Environment variable: OTEL_EXPORTER_PROMETHEUS_PORT + port: 9464 + # Configure a periodic metric reader. + - periodic: + # Configure delay interval (in milliseconds) between start of two consecutive exports. + # + # Environment variable: OTEL_METRIC_EXPORT_INTERVAL + interval: 5000 + # Configure maximum allowed time (in milliseconds) to export data. + # + # Environment variable: OTEL_METRIC_EXPORT_TIMEOUT + timeout: 30000 + # Configure exporter. + # + # Environment variable: OTEL_METRICS_EXPORTER + exporter: + # Configure exporter to be OTLP. + otlp: + # Configure protocol. + # + # Environment variable: OTEL_EXPORTER_OTLP_PROTOCOL, OTEL_EXPORTER_OTLP_METRICS_PROTOCOL + protocol: http/protobuf + # Configure endpoint. + # + # Environment variable: OTEL_EXPORTER_OTLP_ENDPOINT, OTEL_EXPORTER_OTLP_METRICS_ENDPOINT + endpoint: http://localhost:4318 + # Configure certificate. + # + # Environment variable: OTEL_EXPORTER_OTLP_CERTIFICATE, OTEL_EXPORTER_OTLP_METRICS_CERTIFICATE + certificate: /app/cert.pem + # Configure mTLS private client key. + # + # Environment variable: OTEL_EXPORTER_OTLP_CLIENT_KEY, OTEL_EXPORTER_OTLP_METRICS_CLIENT_KEY + client_key: /app/cert.pem + # Configure mTLS client certificate. + # + # Environment variable: OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE, OTEL_EXPORTER_OTLP_METRICS_CLIENT_CERTIFICATE + client_certificate: /app/cert.pem + # Configure headers. + # + # Environment variable: OTEL_EXPORTER_OTLP_HEADERS, OTEL_EXPORTER_OTLP_METRICS_HEADERS + headers: + api-key: !!str 1234 + # Configure compression. + # + # Environment variable: OTEL_EXPORTER_OTLP_COMPRESSION, OTEL_EXPORTER_OTLP_METRICS_COMPRESSION + compression: gzip + # Configure max time (in milliseconds) to wait for each export. + # + # Environment variable: OTEL_EXPORTER_OTLP_TIMEOUT, OTEL_EXPORTER_OTLP_METRICS_TIMEOUT + timeout: 10000 + # Configure temporality preference. + # + # Environment variable: OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE + temporality_preference: delta + # Configure default histogram aggregation. + # + # Environment variable: OTEL_EXPORTER_OTLP_METRICS_DEFAULT_HISTOGRAM_AGGREGATION + default_histogram_aggregation: base2_exponential_bucket_histogram + # Configure a periodic metric reader. + - periodic: + # Configure exporter. + exporter: + # Configure exporter to be console. + console: {} + # Configure views. Each view has a selector which determines the instrument(s) it applies to, and a configuration for the resulting stream(s). + views: + # Configure a view. + - selector: + # Configure instrument name selection criteria. + instrument_name: my-instrument + # Configure instrument type selection criteria. + instrument_type: histogram + # Configure the instrument unit selection criteria. + unit: ms + # Configure meter name selection criteria. + meter_name: my-meter + # Configure meter version selection criteria. + meter_version: 1.0.0 + # Configure meter schema url selection criteria. + meter_schema_url: https://opentelemetry.io/schemas/1.16.0 + # Configure stream. + stream: + # Configure metric name of the resulting stream(s). + name: new_instrument_name + # Configure metric description of the resulting stream(s). + description: new_description + # Configure aggregation of the resulting stream(s). Known values include: default, drop, explicit_bucket_histogram, base2_exponential_bucket_histogram, last_value, sum. + aggregation: + # Configure aggregation to be explicit_bucket_histogram. + explicit_bucket_histogram: + # Configure bucket boundaries. + boundaries: [ 0.0, 5.0, 10.0, 25.0, 50.0, 75.0, 100.0, 250.0, 500.0, 750.0, 1000.0, 2500.0, 5000.0, 7500.0, 10000.0 ] + # Configure record min and max. + record_min_max: true + # Configure attribute keys retained in the resulting stream(s). + attribute_keys: + - key1 + - key2 + +# Configure text map context propagators. +# +# Environment variable: OTEL_PROPAGATORS +propagator: + composite: [tracecontext, baggage, b3, b3multi, jaeger, xray, ottrace] + +# Configure tracer provider. +tracer_provider: + # Configure span processors. + processors: + # Configure a batch span processor. + - batch: + # Configure delay interval (in milliseconds) between two consecutive exports. + # + # Environment variable: OTEL_BSP_SCHEDULE_DELAY + schedule_delay: 5000 + # Configure maximum allowed time (in milliseconds) to export data. + # + # Environment variable: OTEL_BSP_EXPORT_TIMEOUT + export_timeout: 30000 + # Configure maximum queue size. + # + # Environment variable: OTEL_BSP_MAX_QUEUE_SIZE + max_queue_size: 2048 + # Configure maximum batch size. + # + # Environment variable: OTEL_BSP_MAX_EXPORT_BATCH_SIZE + max_export_batch_size: 512 + # Configure exporter. + # + # Environment variable: OTEL_TRACES_EXPORTER + exporter: + # Configure exporter to be OTLP. + otlp: + # Configure protocol. + # + # Environment variable: OTEL_EXPORTER_OTLP_PROTOCOL, OTEL_EXPORTER_OTLP_TRACES_PROTOCOL + protocol: http/protobuf + # Configure endpoint. + # + # Environment variable: OTEL_EXPORTER_OTLP_ENDPOINT, OTEL_EXPORTER_OTLP_TRACES_ENDPOINT + endpoint: http://localhost:4318 + # Configure certificate. + # + # Environment variable: OTEL_EXPORTER_OTLP_CERTIFICATE, OTEL_EXPORTER_OTLP_TRACES_CERTIFICATE + certificate: /app/cert.pem + # Configure mTLS private client key. + # + # Environment variable: OTEL_EXPORTER_OTLP_CLIENT_KEY, OTEL_EXPORTER_OTLP_TRACES_CLIENT_KEY + client_key: /app/cert.pem + # Configure mTLS client certificate. + # + # Environment variable: OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE, OTEL_EXPORTER_OTLP_TRACES_CLIENT_CERTIFICATE + client_certificate: /app/cert.pem + # Configure headers. + # + # Environment variable: OTEL_EXPORTER_OTLP_HEADERS, OTEL_EXPORTER_OTLP_TRACES_HEADERS + headers: + api-key: !!str 1234 + # Configure compression. + # + # Environment variable: OTEL_EXPORTER_OTLP_COMPRESSION, OTEL_EXPORTER_OTLP_TRACES_COMPRESSION + compression: gzip + # Configure max time (in milliseconds) to wait for each export. + # + # Environment variable: OTEL_EXPORTER_OTLP_TIMEOUT, OTEL_EXPORTER_OTLP_TRACES_TIMEOUT + timeout: 10000 + # Configure a batch span processor. + - batch: + # Configure exporter. + # + # Environment variable: OTEL_TRACES_EXPORTER + exporter: + # Configure exporter to be zipkin. + zipkin: + # Configure endpoint. + # + # Environment variable: OTEL_EXPORTER_ZIPKIN_ENDPOINT + endpoint: http://localhost:9411/api/v2/spans + # Configure max time (in milliseconds) to wait for each export. + # + # Environment variable: OTEL_EXPORTER_ZIPKIN_TIMEOUT + timeout: 10000 + # Configure a simple span processor. + - simple: + # Configure exporter. + exporter: + # Configure exporter to be console. + console: {} + # Configure span limits. See also attribute_limits. + limits: + # Configure max span attribute value size. Overrides attribute_limits.attribute_value_length_limit. + # + # Environment variable: OTEL_SPAN_ATTRIBUTE_VALUE_LENGTH_LIMIT + attribute_value_length_limit: 4096 + # Configure max span attribute count. Overrides attribute_limits.attribute_count_limit. + # + # Environment variable: OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT + attribute_count_limit: 128 + # Configure max span event count. + # + # Environment variable: OTEL_SPAN_EVENT_COUNT_LIMIT + event_count_limit: 128 + # Configure max span link count. + # + # Environment variable: OTEL_SPAN_LINK_COUNT_LIMIT + link_count_limit: 128 + # Configure max attributes per span event. + # + # Environment variable: OTEL_EVENT_ATTRIBUTE_COUNT_LIMIT + event_attribute_count_limit: 128 + # Configure max attributes per span link. + # + # Environment variable: OTEL_LINK_ATTRIBUTE_COUNT_LIMIT + link_attribute_count_limit: 128 + # Configure the sampler. + sampler: + # Configure sampler to be parent_based. Known values include: always_off, always_on, jaeger_remote, parent_based, trace_id_ratio_based. + # + # Environment variable: OTEL_TRACES_SAMPLER=parentbased_* + parent_based: + # Configure root sampler. + # + # Environment variable: OTEL_TRACES_SAMPLER=parentbased_traceidratio + root: + # Configure sampler to be trace_id_ratio_based. + trace_id_ratio_based: + # Configure trace_id_ratio. + # + # Environment variable: OTEL_TRACES_SAMPLER_ARG=traceidratio=0.0001 + ratio: 0.0001 + # Configure remote_parent_sampled sampler. + remote_parent_sampled: + # Configure sampler to be always_on. + always_on: {} + # Configure remote_parent_not_sampled sampler. + remote_parent_not_sampled: + # Configure sampler to be always_off. + always_off: {} + # Configure local_parent_sampled sampler. + local_parent_sampled: + # Configure sampler to be always_on. + always_on: {} + # Configure local_parent_not_sampled sampler. + local_parent_not_sampled: + parent_based: + remote_parent_not_sampled: + trace_id_ratio_based: + ratio: 0.0001 + +# Configure resource for all signals. +resource: + # Configure resource attributes. + # + # Environment variable: OTEL_RESOURCE_ATTRIBUTES + attributes: + # Configure `service.name` resource attribute + # + # Environment variable: OTEL_SERVICE_NAME + service.name: !!str "unknown_service" + # Configure the resource schema URL. + schema_url: https://opentelemetry.io/schemas/1.16.0 diff --git a/prototypes/python/tests/test_configuration.py b/prototypes/python/tests/test_configuration.py index cbe1f38..dbbb390 100644 --- a/prototypes/python/tests/test_configuration.py +++ b/prototypes/python/tests/test_configuration.py @@ -20,8 +20,13 @@ create_object, validate_configuration, load_configuration, + substitute_environment_variables, + render_schema, ) +from unittest.mock import patch +from os import environ from pathlib import Path +from pytest import fail data_path = Path(__file__).parent.joinpath("data") @@ -29,10 +34,13 @@ def test_create_object(): configuration = load_configuration( - data_path.joinpath("kitchen-sink.yaml") + data_path.joinpath("configuration_0.yaml") ) - validate_configuration(configuration) + try: + validate_configuration(configuration) + except Exception as error: + fail(f"Unexpected exception raised: {error}") processed_schema = process_schema( resolve_schema( @@ -113,3 +121,44 @@ def test_create_object(): _resource. _schema_url ) == "https://opentelemetry.io/schemas/1.16.0" + + +@patch.dict(environ, {"OTEL_BLRB_EXPORT_TIMEOUT": "943"}, clear=True) +def test_substitute_environment_variables(): + configuration = load_configuration( + data_path.joinpath("configuration_1.yaml") + ) + + processed_schema = process_schema( + resolve_schema( + data_path.joinpath("opentelemetry_configuration.json") + ) + ) + configuration = substitute_environment_variables( + configuration, processed_schema + ) + + assert ( + configuration + ["logger_provider"] + ["processors"] + [0] + ["batch"] + ["export_timeout"] + ) == 943 + try: + validate_configuration(configuration) + except Exception as error: + fail(f"Unexpected exception raised: {error}") + + +def test_render(tmpdir): + + render_schema( + process_schema( + resolve_schema( + data_path.joinpath("opentelemetry_configuration.json") + ) + ), + tmpdir.join("path_function.py") + )