diff --git a/.github/workflows/publish-dockerhub.yml b/.github/workflows/publish-dockerhub.yml index 6b95333..9814c35 100644 --- a/.github/workflows/publish-dockerhub.yml +++ b/.github/workflows/publish-dockerhub.yml @@ -8,31 +8,32 @@ on: jobs: buildx: runs-on: ubuntu-latest + environment: dockerhub steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: ref: master - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 with: platforms: all - name: Set up Docker Buildx id: buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 with: version: latest - name: Login to DockerHub - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: username: ${{ secrets.DOCKER_HUB_USER }} password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} - name: Build and push id: docker_build - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v5 with: push: true platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v6 diff --git a/.github/workflows/publish-pypi.yml b/.github/workflows/publish-pypi.yml index 15a12ec..f93394a 100644 --- a/.github/workflows/publish-pypi.yml +++ b/.github/workflows/publish-pypi.yml @@ -7,13 +7,18 @@ jobs: build-n-publish: name: Build and publish Python 🐍 distributions 📦 to PyPI and TestPyPI runs-on: ubuntu-latest - + environment: pypi + permissions: + # IMPORTANT: this permission is mandatory for trusted publishing + id-token: write steps: - - uses: actions/checkout@v3 + + - uses: actions/checkout@v4 with: ref: master + - name: Set up Python 3.10 - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: '3.10' @@ -27,7 +32,4 @@ jobs: python setup.py sdist bdist_wheel - name: Publish distribution to PyPI - uses: pypa/gh-action-pypi-publish@master - with: - user: __token__ - password: ${{ secrets.pypi_api_key }} + uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/.github/workflows/run_tox.yml b/.github/workflows/run_tox.yml index e13878f..1c3d54c 100644 --- a/.github/workflows/run_tox.yml +++ b/.github/workflows/run_tox.yml @@ -7,8 +7,8 @@ jobs: name: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: python-version: '3.10' - uses: pre-commit/action@v3.0.0 @@ -20,12 +20,12 @@ jobs: strategy: max-parallel: 4 matrix: - python-version: ['3.8', '3.9', '3.10', '3.11'] + python-version: ['3.10', '3.11', '3.12'] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Install dependencies diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e2f1f7a..d069fb3 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -6,40 +6,17 @@ repos: - id: check-builtin-literals - id: check-docstring-first - id: check-merge-conflict -# - id: check-toml + - id: check-toml - id: check-yaml - id: debug-statements - id: end-of-file-fixer - id: trailing-whitespace - - repo: https://github.com/pycqa/isort - rev: 5.12.0 + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.3.4 hooks: - - id: isort - name: isort (python) - - - - repo: https://github.com/PyCQA/flake8 - rev: '6.0.0' - hooks: - - id: flake8 - additional_dependencies: - - flake8-bugbear==23.2.13 - - flake8-comprehensions==3.10.1 - - flake8-pytest-style==1.6 -# - flake8-spellcheck==0.28 -# - flake8-unused-arguments==0.0.12 - - flake8-noqa==1.3 - - pep8-naming==0.13.3 - - - - repo: https://github.com/asottile/pyupgrade - rev: v3.3.1 - hooks: - - id: pyupgrade - args: ["--py38-plus"] - + - id: ruff - repo: https://github.com/pre-commit/pygrep-hooks rev: v1.10.0 diff --git a/.ruff.toml b/.ruff.toml new file mode 100644 index 0000000..f60ff22 --- /dev/null +++ b/.ruff.toml @@ -0,0 +1,77 @@ + +line-length = 120 +indent-width = 4 + +target-version = "py310" +src = ["src", "test"] + +[lint] + +# https://docs.astral.sh/ruff/settings/#ignore-init-module-imports +ignore-init-module-imports = true + +select = [ + "E", "W", # https://docs.astral.sh/ruff/rules/#pycodestyle-e-w + "I", # https://docs.astral.sh/ruff/rules/#isort-i + "UP", # https://docs.astral.sh/ruff/rules/#pyupgrade-up + + "A", # https://docs.astral.sh/ruff/rules/#flake8-builtins-a + "ASYNC", # https://docs.astral.sh/ruff/rules/#flake8-async-async + "C4", # https://docs.astral.sh/ruff/rules/#flake8-comprehensions-c4 + "EM", # https://docs.astral.sh/ruff/rules/#flake8-errmsg-em + "FIX", # https://docs.astral.sh/ruff/rules/#flake8-fixme-fix + "INP", # https://docs.astral.sh/ruff/rules/#flake8-no-pep420-inp + "ISC", # https://docs.astral.sh/ruff/rules/#flake8-implicit-str-concat-isc + "PIE", # https://docs.astral.sh/ruff/rules/#flake8-pie-pie + "PT", # https://docs.astral.sh/ruff/rules/#flake8-pytest-style-pt + "PTH", # https://docs.astral.sh/ruff/rules/#flake8-use-pathlib-pth + "RET", # https://docs.astral.sh/ruff/rules/#flake8-return-ret + "SIM", # https://docs.astral.sh/ruff/rules/#flake8-simplify-sim + "SLOT", # https://docs.astral.sh/ruff/rules/#flake8-slots-slot + "T10", # https://docs.astral.sh/ruff/rules/#flake8-debugger-t10 + "TCH", # https://docs.astral.sh/ruff/rules/#flake8-type-checking-tch + "TD", # https://docs.astral.sh/ruff/rules/#flake8-todos-td + + "TRY", # https://docs.astral.sh/ruff/rules/#tryceratops-try + "FLY", # https://docs.astral.sh/ruff/rules/#flynt-fly + "PERF", # https://docs.astral.sh/ruff/rules/#perflint-perf + "RUF", # https://docs.astral.sh/ruff/rules/#ruff-specific-rules-ruf + + # "PL", # https://docs.astral.sh/ruff/rules/#pylint-pl + # "FURB", # https://docs.astral.sh/ruff/rules/#refurb-furb +] + +ignore = [ + "RET501", # https://docs.astral.sh/ruff/rules/unnecessary-return-none/#unnecessary-return-none-ret501 + "TRY400", # https://docs.astral.sh/ruff/rules/error-instead-of-exception/ + + "A003", # https://docs.astral.sh/ruff/rules/builtin-attribute-shadowing/ + + "UP038", # https://docs.astral.sh/ruff/rules/non-pep604-isinstance/ +] + + +[format] +# Use single quotes for non-triple-quoted strings. +quote-style = "single" + + + +[lint.flake8-builtins] +builtins-ignorelist = ["id", "input"] + + +[lint.per-file-ignores] +"docs/conf.py" = ["INP001", "A001"] +"setup.py" = ["PTH123"] +"tests/*" = [ + "ISC002", # Implicitly concatenated string literals over multiple lines + "E501", # Line too long + "INP001", # File `FILE_NAME` is part of an implicit namespace package. Add an `__init__.py` +] + + + +[lint.isort] +# https://docs.astral.sh/ruff/settings/#lint_isort_lines-after-imports +lines-after-imports = 2 diff --git a/Dockerfile b/Dockerfile index e442291..e3300f5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.10-alpine +FROM python:3.11-alpine VOLUME /sml2mqtt diff --git a/docs/cli.rst b/docs/cli.rst new file mode 100644 index 0000000..e5cf632 --- /dev/null +++ b/docs/cli.rst @@ -0,0 +1,11 @@ +************************************** +Command Line Interface +************************************** + +.. _COMMAND_LINE_INTERFACE: + +.. exec_code:: + :hide_code: + + import sml2mqtt.__args__ + sml2mqtt.__args__.get_command_line_args(['-h']) diff --git a/docs/conf.py b/docs/conf.py index 09bc8b8..7fa10c2 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,6 +1,8 @@ import os import re import sys +from pathlib import Path + RTD_BUILD = os.environ.get('READTHEDOCS') == 'True' @@ -50,7 +52,7 @@ autoclass_content = 'class' # so autodoc does find the source -sys.path.insert(0, os.path.join(os.path.abspath('..'), 'src')) +sys.path.insert(0, str(Path(__file__).parent.with_name('src'))) # -- Options for autodoc pydantic ------------------------------------------------- @@ -86,7 +88,8 @@ # Don't show warnings for missing python references since these are created via intersphinx during the RTD build if not RTD_BUILD: nitpick_ignore_regex = [ - (re.compile(r'^py:class'), re.compile(r'pathlib\..+')), (re.compile(r'^py:data'), re.compile(r'typing\..+')), - (re.compile(r'^py:class'), re.compile(r'pydantic\..+|.+Constrained(?:Str|Int)Value')), + (re.compile(r'^py:class'), re.compile(r'pydantic_core\..+')), + # WARNING: py:class reference target not found: sml2mqtt.config.operations.Annotated + (re.compile(r'^py:class'), re.compile(r'.+\.Annotated')), ] diff --git a/docs/configuration.rst b/docs/configuration.rst index cc4bfcb..d6a0363 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -1,168 +1,127 @@ -************************************** -Configuration & CLI -************************************** - -.. _COMMAND_LINE_INTERFACE: - -Command Line Interface -====================================== - -.. exec_code:: - :hide_code: - - import sml2mqtt.__args__ - sml2mqtt.__args__.get_command_line_args(['-h']) +.. py:currentmodule:: sml2mqtt.config.config +************************************** Configuration -====================================== +************************************** -Configuration is done through ``config.yml`` The parent folder of the file can be specified with ``-c PATH`` or ``--config PATH``. -If nothing is specified the file ``config.yml`` is searched in the subdirectory ``sml2mqtt`` in +Configuration of sml2mqtt is done through a yaml file. +The path to the file can be specified with ``-c PATH`` or ``--config PATH``. +If nothing is specified a file with the name ``config.yml`` is searched in the subdirectory ``sml2mqtt`` in * the current working directory * the venv directory * the user home -If the config is specified and it does not yet exist a default configuration file will be created +If a config file is specified and it does not yet exist a default configuration file will be created. Example --------------------------------------- +====================================== + + + +.. + YamlModel: Settings .. code-block:: yaml logging: - level: INFO # Log level - file: sml2mqtt.log # Log file path (absolute or relative to config file) + level: INFO # Log level + file: sml2mqtt.log # Log file path (absolute or relative to config file) or stdout mqtt: connection: - client id: sml2mqtt + identifier: sml2mqtt-ZqlFvhSBdDGvJ host: localhost port: 1883 user: '' password: '' - tls: false - tls insecure: false - - # MQTT default configuration - # All other topics use these values if no other values for qos/retain are set - # It's possible to override - # - topic (fragment that is used to build the full mqtt topic) - # - full_topic (will not build the topic from the fragments but rather use the configured value) - # - qos - # - retain - # for each (!) mqtt-topic entry - defaults: - qos: 0 - retain: false topic prefix: sml2mqtt - + defaults: + qos: 0 # Default value for QOS if no other QOS value in the config entry is set + retain: false # Default value for retain if no other retain value in the config entry is set last will: - topic: status + topic: status # Topic fragment for building this topic with the parent topic general: - Wh in kWh: true # Automatically convert Wh to kWh - republish after: 120 # Republish automatically after this time (if no other every filter is configured) - - # Serial port configurations for the sml readers - ports: - - url: COM1 - timeout: 3 - - url: /dev/ttyS0 - timeout: 3 + Wh in kWh: true # Automatically convert Wh to kWh + republish after: 120 # Republish automatically after this time (if no other filter configured) + inputs: + - type: serial + url: COM1 # Device path + timeout: 3 # Seconds after which a timeout will be detected (default=3) + - type: serial + url: /dev/ttyS0 # Device path + timeout: 3 # Seconds after which a timeout will be detected (default=3) devices: - # Device configuration by OBIS value 0100000009ff or by url if the device does not report OBIS 0100000009ff - 11111111111111111111: - mqtt: - topic: DEVICE_TOPIC - - # OBIS IDs that will not be processed (optional) - skip: - - OBIS - - values - - to skip - - # Configuration how each OBIS value is reported. Create as many OBIS IDs (e.g. 0100010800ff as you like). - # Each sub entry (mqtt, workarounds, transformations, filters) is optional and can be omitted - values: + # Device configuration by reported id + device_id_hex: - OBIS: - # Sub topic how this value is reported. - mqtt: - topic: OBIS + mqtt: # Optional MQTT configuration for this meter. + topic: DEVICE_BASE_TOPIC # Topic fragment for building this topic with the parent topic - # Workarounds allow the enabling workarounds (e.g. if the device has strange behaviour) - # These are the available workarounds - workarounds: - - negative on energy meter status: true # activate this workaround + status: # Optional MQTT status topic configuration for this meter + topic: status # Topic fragment for building this topic with the parent topic - # Transformations allow mathematical calculations on the obis value - # They are applied in order how they are defined - transformations: - - factor: 3 # multiply with factor - - offset: 100 # add offset - - round: 2 # round on two digits + skip: # OBIS codes (HEX) of values that will not be published (optional) + - '00112233445566' - # Filters control how often a value is published over mqtt. - # If one filter is true the value will be published - filters: - - diff: 10 # report if value difference is >= 10 - - perc: 10 # report if percentage change is >= 10% - - every: 120 # report at least every 120 secs (overrides the value from general) + # Configurations for each of the values (optional) + values: + - obis: '00112233445566' # Obis code for this value + mqtt: # Mqtt config for this value (optional) + topic: OBIS_VALUE_TOPIC # Topic fragment for building this topic with the topic prefix + # A sequence of operations that will be evaluated one after another. + # If one operation blocks nothing will be reported for this frame + operations: + - negative on energy meter status: true # Make value negative based on an energy meter status. Set to "true" to enable or to "false" to disable workaround. If the default obis code for the energy meter is wrong set to the appropriate meter obis code instead + - factor: 3 # Factor with which the value gets multiplied + - offset: 100 # Offset that gets added on the value + - round: 2 # Round to the specified digits + - refresh action: 300 # Republish value every 300s -Example devices --------------------------------------- -One energy meter is connected to the serial port. The serial meter reports OBIS ``0100000009ff`` -as ``11111111111111111111``. +Example input Tibber bridge +====================================== -For this device +These input settings can be used to poll data from a Tibber bridge: -* the mqtt topic fragment is set to ``light`` -* the value ``0100010801ff`` will not be published -* The following values of the device are specially configured: +.. + YamlModel: Settings + + +.. code-block:: yaml - * Energy value (OBIS ``0100010800ff``) + inputs: + - type: http + url: http://IP_OR_HOSTNAME_OF_TIBBER_BRIDGE/data.json?node_id=1 + interval: 3 # Poll interval secs + timeout: 10 # After which time the input will change to TIMEOUT + user: "admin" + password: "printed on bridge socket" - * Will be rounded to one digit - * Will be published on change **or** at least every hour - * The mqtt topic used is ``sml2mqtt/light/energy``. - (Built through ``topic prefix`` + ``device mqtt`` + ``value mqtt``) +Example mqtt config +====================================== - * Power value (OBIS ``0100100700ff``) +MQTT topics can be configured either by providing a full topic or a topic fragment. +With a topic fragment the resulting topic is build with the parent topic. +The structure is ``topic prefix`` / ``device`` / ``value``. +Providing a full topic will ignore the fragments. +The entries for qos and retain are optional. - * Will be rounded to one digit - * Will be published if at least a 5% power change occurred **or** at least every 2 mins - (default from ``general`` -> ``republish after``) - * The mqtt topic used is ``sml2mqtt/light/power`` +.. + YamlModel: OptionalMqttPublishConfig .. code-block:: yaml - devices: - 11111111111111111111: - mqtt: - topic: light - skip: - - 0100010801ff - values: - 0100010800ff: - mqtt: - topic: energy - transformations: - - round: 1 - filters: - - every: 3600 - 0100100700ff: - mqtt: - topic: power - filters: - - perc: 5 + full topic: my/full/topic + qos: 1 + Configuration Reference @@ -176,16 +135,51 @@ logging -------------------------------------- .. autopydantic_model:: sml2mqtt.config.logging.LoggingSettings + :exclude-members: set_log_level + + +.. _CONFIG_GENERAL: general -------------------------------------- .. autopydantic_model:: sml2mqtt.config.config.GeneralSettings -ports + +.. _CONFIG_INPUTS: + +inputs -------------------------------------- -.. autopydantic_model:: sml2mqtt.config.config.PortSettings +.. autopydantic_model:: sml2mqtt.config.inputs.SerialSourceSettings + :exclude-members: get_device_name + +Example: + +.. + YamlModel: sml2mqtt.config.inputs.SerialSourceSettings + +.. code-block:: yaml + + type: serial + url: COM3 + + +.. autopydantic_model:: sml2mqtt.config.inputs.HttpSourceSettings + :exclude-members: get_device_name, get_request_timeout + +Example: + +.. + YamlModel: sml2mqtt.config.inputs.HttpSourceSettings + +.. code-block:: yaml + + type: http + url: http://localhost:8080/sml + interval: 3 + timeout: 10 + mqtt -------------------------------------- @@ -200,6 +194,10 @@ mqtt .. autopydantic_model:: MqttDefaultPublishConfig +.. autopydantic_model:: sml2mqtt.config.mqtt_tls.MqttTlsOptions + :exclude-members: get_client_kwargs + + devices -------------------------------------- @@ -208,12 +206,3 @@ devices .. autopydantic_model:: SmlDeviceConfig .. autopydantic_model:: SmlValueConfig - -.. autoclass:: WorkaroundOptionEnum - :members: - -.. autoclass:: TransformOptionEnum - :members: - -.. autoclass:: FilterOptionEnum - :members: diff --git a/docs/getting_started.rst b/docs/getting_started.rst index befcc3c..c417b2a 100644 --- a/docs/getting_started.rst +++ b/docs/getting_started.rst @@ -2,22 +2,171 @@ Getting started ************************************** +1. Installation +====================================== + First install ``sml2mqtt`` e.g in a :ref:`virtual environment `. +2. Create default configuration +====================================== + Run ``sml2mqtt`` with a path to a configuration file. A new default configuration file will be created. -Edit the configuration file and add the serial ports. +.. py:currentmodule:: sml2mqtt.config.config + + +.. + YamlModel: Settings + +.. code-block:: yaml + + logging: + level: INFO # Log level + file: sml2mqtt.log # Log file path (absolute or relative to config file) or stdout + + mqtt: + connection: + identifier: sml2mqtt-ZqlFvhSBdDGvJ + host: localhost + port: 1883 + user: '' + password: '' + topic prefix: sml2mqtt + defaults: + qos: 0 # Default value for QOS if no other QOS value in the config entry is set + retain: false # Default value for retain if no other retain value in the config entry is set + last will: + topic: status # Topic fragment for building this topic with the parent topic + + general: + Wh in kWh: true # Automatically convert Wh to kWh + republish after: 120 # Republish automatically after this time (if no other filter configured) + + inputs: + - type: serial + url: COM1 # Device path + timeout: 3 # Seconds after which a timeout will be detected (default=3) + - type: serial + url: /dev/ttyS0 # Device path + timeout: 3 # Seconds after which a timeout will be detected (default=3) + + devices: + # Device configuration by reported id + device_id_hex: + + mqtt: # Optional MQTT configuration for this meter. + topic: DEVICE_BASE_TOPIC # Topic fragment for building this topic with the parent topic + + status: # Optional MQTT status topic configuration for this meter + topic: status # Topic fragment for building this topic with the parent topic + + skip: # OBIS codes (HEX) of values that will not be published (optional) + - '00112233445566' + + # Configurations for each of the values (optional) + values: + + - obis: '00112233445566' # Obis code for this value + mqtt: # Mqtt config for this value (optional) + topic: OBIS_VALUE_TOPIC # Topic fragment for building this topic with the topic prefix + # A sequence of operations that will be evaluated one after another. + # If one operation blocks nothing will be reported for this frame + operations: + - negative on energy meter status: true # Make value negative based on an energy meter status. Set to "true" to enable or to "false" to disable workaround. If the default obis code for the energy meter is wrong set to the appropriate meter obis code instead + - factor: 3 # Factor with which the value gets multiplied + - offset: 100 # Offset that gets added on the value + - round: 2 # Round to the specified digits + - refresh action: 300 # Republish value every 300s + +3. Edit inputs and mqtt +====================================== + +Edit the configuration file and configure the appropriate :ref:`inputs ` for +serial or http (e.g. for tibber) and edit the mqtt settings. -Now run ``sml2mqtt`` with the path to the configuration file and the ``analyze`` option. + +.. + YamlModel: Settings + +.. code-block:: yaml + :linenos: + :emphasize-lines: 8-11, 24-29 + + + logging: + level: INFO # Log level + file: sml2mqtt.log # Log file path (absolute or relative to config file) or stdout + + mqtt: + connection: + identifier: sml2mqtt-ZqlFvhSBdDGvJ + host: localhost + port: 1883 + user: '' + password: '' + topic prefix: sml2mqtt + defaults: + qos: 0 # Default value for QOS if no other QOS value in the config entry is set + retain: false # Default value for retain if no other retain value in the config entry is set + last will: + topic: status # Topic fragment for building this topic with the parent topic + + general: + Wh in kWh: true # Automatically convert Wh to kWh + republish after: 120 # Republish automatically after this time (if no other filter configured) + + inputs: + - type: serial + url: COM1 # Device path + timeout: 3 # Seconds after which a timeout will be detected (default=3) + - type: serial + url: /dev/ttyS0 # Device path + timeout: 3 # Seconds after which a timeout will be detected (default=3) + + devices: + # Device configuration by reported id + device_id_hex: + + mqtt: # Optional MQTT configuration for this meter. + topic: DEVICE_BASE_TOPIC # Topic fragment for building this topic with the parent topic + + status: # Optional MQTT status topic configuration for this meter + topic: status # Topic fragment for building this topic with the parent topic + + skip: # OBIS codes (HEX) of values that will not be published (optional) + - '00112233445566' + + # Configurations for each of the values (optional) + values: + + - obis: '00112233445566' # Obis code for this value + mqtt: # Mqtt config for this value (optional) + topic: OBIS_VALUE_TOPIC # Topic fragment for building this topic with the topic prefix + # A sequence of operations that will be evaluated one after another. + # If one operation blocks nothing will be reported for this frame + operations: + - negative on energy meter status: true # Make value negative based on an energy meter status. Set to "true" to enable or to "false" to disable workaround. If the default obis code for the energy meter is wrong set to the appropriate meter obis code instead + - factor: 3 # Factor with which the value gets multiplied + - offset: 100 # Offset that gets added on the value + - round: 2 # Round to the specified digits + - refresh action: 300 # Republish value every 300s + + +4. Run with analyze +====================================== + +Now run ``sml2mqtt`` with the path to the configuration file and the ``--analyze`` option. (see :ref:`command line interface `). This will process one sml frame from the meter and report the output. It's a convenient way to check what values will be reported. -It will also show how the configuration changes the sml values (e.g. if you add a transformation or a workaround). +It will also show how the configuration changes the reported values when you add an operation. +Check if the meter reports the serial number unter obis ``0100000009ff``. Example output for the meter data: .. code-block:: text + :emphasize-lines: 33, 38 SmlMessage transaction_id: 17c77d6b @@ -116,9 +265,141 @@ Example output for the meter data: crc16 : 56696 -Check if the meter reports the serial number unter obis ``0100000009ff``. -If not it's possible to configure another number (of even multiple ones) for configuration matching. -If yes replace ``SERIAL_ID_HEX`` in the dummy configuration with the reported -serial number (here ``11111111111111111111``). -Modify the device configuration to your liking (see configuration documentation). -Run the analyze command again and see how the output changes and observe the reported values. +If the meter does not report ``0100000009ff`` it's possible to configure another number (of even multiple ones) +for configuration matching (see :ref:`general under config `). + +5. Edit device settings +====================================== + +Replace ``device_id_hex`` in the dummy configuration with the reported number (here ``11111111111111111111``). +Edit the mqtt settings or remove them to use the default. Add the obis code of values that should not be reported +to the skip section. Run the analyze command again to see how the reported values change. + +.. + YamlModel: Settings + +.. code-block:: yaml + :linenos: + :emphasize-lines: 13, 15-16, 18-19, 21-22 + + # ... + + inputs: + - type: serial + url: COM1 # Device path + timeout: 3 # Seconds after which a timeout will be detected (default=3) + - type: serial + url: /dev/ttyS0 # Device path + timeout: 3 # Seconds after which a timeout will be detected (default=3) + + devices: + # Device configuration by reported id + '11111111111111111111': + + mqtt: # Optional MQTT configuration for this meter. + topic: meter_light # Topic fragment for building this topic with the parent topic + + status: # Optional MQTT status topic configuration for this meter + topic: status # Topic fragment for building this topic with the parent topic + + skip: # OBIS codes (HEX) of values that will not be published (optional) + - '8181c78205ff' + + # Configurations for each of the values (optional) + values: + + - obis: '00112233445566' # Obis code for this value + mqtt: # Mqtt config for this value (optional) + topic: OBIS_VALUE_TOPIC # Topic fragment for building this topic with the topic prefix + # A sequence of operations that will be evaluated one after another. + # If one operation blocks nothing will be reported for this frame + operations: + - negative on energy meter status: true # Make value negative based on an energy meter status. Set to "true" to enable or to "false" to disable workaround. If the default obis code for the energy meter is wrong set to the appropriate meter obis code instead + - factor: 3 # Factor with which the value gets multiplied + - offset: 100 # Offset that gets added on the value + - round: 2 # Round to the specified digits + - refresh action: 300 # Republish value every 300s + + +6. Edit value settings +====================================== + +It's possible to further configure how values will be reported. +For every value there are multiple operations that can be applied. +Each sml value can also be processed multiple times. + +Run the analyze command again to see how the reported values change. + +.. + YamlModel: Settings + +.. code-block:: yaml + :linenos: + :emphasize-lines: 27-37, 39-45, 47-52 + + # ... + + inputs: + - type: serial + url: COM1 # Device path + timeout: 3 # Seconds after which a timeout will be detected (default=3) + - type: serial + url: /dev/ttyS0 # Device path + timeout: 3 # Seconds after which a timeout will be detected (default=3) + + devices: + # Device configuration by reported id + '11111111111111111111': + + mqtt: # Optional MQTT configuration for this meter. + topic: meter_light # Topic fragment for building this topic with the parent topic + + status: # Optional MQTT status topic configuration for this meter + topic: status # Topic fragment for building this topic with the parent topic + + skip: # OBIS codes (HEX) of values that will not be published (optional) + - '8181c78205ff' + + # Configurations for each of the values (optional) + values: + + - obis: '0100010800ff' # Obis code for the energy value + mqtt: + topic: energy_today + operations: + - type: meter # A virtual meter + start now: true # Start immediately + reset times: # Reset at midnight + - 00:00 + - round: 1 + - type: change filter # Only report on changes + - refresh action: 01:00 # ... but refresh every hour + + - obis: '0100010800ff' # Obis code for the energy value + mqtt: + topic: energy_total + operations: + - round: 1 + - type: change filter + - refresh action: 01:00 + + - obis: '0100100700ff' # Obis code for the power value + mqtt: + topic: power + operations: + - type: delta filter + min: 10 + min %: 5 + - refresh action: 01:00 + + +Output from the analyze command that shows what values will be reported + +.. code-block:: text + + ... + sml2mqtt/meter_light/energy_today: 0 (QOS: 0, retain: False) + sml2mqtt/meter_light/energy_total: 12345.7 (QOS: 0, retain: False) + sml2mqtt/meter_light/power: 555 (QOS: 0, retain: False) + sml2mqtt/meter_light/status: OK (QOS: 0, retain: False) + ... diff --git a/docs/index.rst b/docs/index.rst index bef04b1..8c9c328 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -2,13 +2,20 @@ Welcome to sml2mqtt's documentation! ==================================== +sml2mqtt is a asyncio application that can read multiple sml (Smart Message Language) streams +from energy meters and report the values through mqtt. +The meters can be read through serial ports or through http (e.g. Tibber) and the values that +will be reported can be processed in various ways with operations. + .. toctree:: :maxdepth: 2 :caption: Contents: installation configuration + cli getting_started + operations Indices and tables diff --git a/docs/installation.rst b/docs/installation.rst index e5af37a..dec1294 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -47,13 +47,13 @@ Go into folder of virtual environment:: python3 -m pip install --upgrade pip setuptools -Install sml2mqtt +Install sml2mqtt:: python3 -m pip install sml2mqtt #. Run sml2mqtt:: - sml2mqtt --config PATH_TO_CONFIGURAT_FILE + sml2mqtt --config PATH_TO_CONFIGURATION_FILE Upgrading @@ -106,7 +106,7 @@ If your installation is not done in "/opt/sml2mqtt/venv/bin" replace accordingly User=openhab Group=openhab Restart=on-failure - RestartSec=2min + RestartSec=10min ExecStart=/opt/sml2mqtt/venv/bin/sml2mqtt -c PATH_TO_CONFIGURATION_FILE [Install] @@ -140,3 +140,6 @@ Installation through `docker `). diff --git a/docs/operations.rst b/docs/operations.rst new file mode 100644 index 0000000..883fe94 --- /dev/null +++ b/docs/operations.rst @@ -0,0 +1,445 @@ +.. py:currentmodule:: sml2mqtt.config.operations + + +************************************** +Operations +************************************** + +It's possible to define operations which are used to process the received value + + + +Filters +====================================== + + +Change Filter +-------------------------------------- + +.. autopydantic_model:: OnChangeFilter + :exclude-members: get_kwargs_on_change + :inherited-members: BaseModel + +Example +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.. + YamlModel: OnChangeFilter + +.. code-block:: yaml + + type: change filter + + +Range filter +-------------------------------------- + +.. autopydantic_model:: RangeFilter + :inherited-members: BaseModel + :exclude-members: type, get_kwargs_limit + +Example +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.. + YamlModel: RangeFilter + +.. code-block:: yaml + + type: range filter + min: 0 + + +Delta Filter +-------------------------------------- + +.. autopydantic_model:: DeltaFilter + :exclude-members: get_kwargs_delta + :inherited-members: BaseModel + +Example +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.. + YamlModel: DeltaFilter + +.. code-block:: yaml + + type: delta filter + min: 5 + min %: 10 + +.. + YamlModel: DeltaFilter + +.. code-block:: yaml + + type: delta filter + min: 5 + +.. + YamlModel: DeltaFilter + +.. code-block:: yaml + + type: delta filter + min %: 10 + + +Throttle Filter +-------------------------------------- + +.. autopydantic_model:: ThrottleFilter + :inherited-members: BaseModel + + +Example +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.. + YamlModel: ThrottleFilter + +.. code-block:: yaml + + throttle filter: 60 + + +Actions +====================================== + + +Refresh Action +-------------------------------------- + +.. autopydantic_model:: RefreshAction + :inherited-members: BaseModel + + +Example +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.. + YamlModel: RefreshAction + +.. code-block:: yaml + + refresh action: 01:30:00 + + +Heartbeat Action +-------------------------------------- + +.. autopydantic_model:: HeartbeatAction + :inherited-members: BaseModel + + +Example +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.. + YamlModel: HeartbeatAction + +.. code-block:: yaml + + heartbeat action: 30 + +Math +====================================== + + +Factor +-------------------------------------- + +.. autopydantic_model:: Factor + :inherited-members: BaseModel + + +Example +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.. + YamlModel: Factor + +.. code-block:: yaml + + factor: -1 + + +Offset +-------------------------------------- + +.. autopydantic_model:: Offset + :inherited-members: BaseModel + + +Example +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.. + YamlModel: Offset + +.. code-block:: yaml + + offset: 10 + +Round +-------------------------------------- + +.. autopydantic_model:: Round + :inherited-members: BaseModel + +Example +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.. + YamlModel: Round + +.. code-block:: yaml + + round: 2 + + +Workarounds +====================================== + + +Negative On Energy Meter Status +-------------------------------------- + +.. autopydantic_model:: NegativeOnEnergyMeterWorkaround + :inherited-members: BaseModel + + +Example +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.. + YamlModel: NegativeOnEnergyMeterWorkaround + +.. code-block:: yaml + + negative on energy meter status: true + + +Date time based +====================================== + + +Virtual Meter +-------------------------------------- + +.. autopydantic_model:: VirtualMeter + :inherited-members: BaseModel + :exclude-members: get_kwargs_dt_fields, type + + +Example +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.. + YamlModel: VirtualMeter + +.. code-block:: yaml + + type: meter + start now: False + reset times: + - 02:00 + reset days: + - 1 + - monday + + +Max Value +-------------------------------------- + +.. autopydantic_model:: MaxValue + :inherited-members: BaseModel + :exclude-members: get_kwargs_dt_fields, type + + +Example +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.. + YamlModel: MaxValue + +.. code-block:: yaml + + type: max value + start now: True + reset times: + - 02:00 + + +Min Value +-------------------------------------- + +.. autopydantic_model:: MinValue + :inherited-members: BaseModel + :exclude-members: get_kwargs_dt_fields, type + + +Example +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.. + YamlModel: MinValue + +.. code-block:: yaml + + type: min value + start now: True + reset times: + - 02:00 + + +Time series +====================================== + + + +Max Value +-------------------------------------- + +.. autopydantic_model:: MaxOfInterval + :inherited-members: BaseModel + :exclude-members: get_kwargs_interval_fields, type + + +Example +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.. + YamlModel: MaxOfInterval + +.. code-block:: yaml + + type: max interval + interval: 3600 + wait for data: False + + +Min Value +-------------------------------------- + +.. autopydantic_model:: MinOfInterval + :inherited-members: BaseModel + :exclude-members: get_kwargs_interval_fields, type + + +Example +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.. + YamlModel: MinOfInterval + +.. code-block:: yaml + + type: min interval + interval: 3600 + wait for data: False + + +Mean Value +-------------------------------------- + +.. autopydantic_model:: MeanOfInterval + :inherited-members: BaseModel + :exclude-members: get_kwargs_interval_fields, type + + +Example +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.. + YamlModel: MeanOfInterval + +.. code-block:: yaml + + type: mean interval + interval: 3600 + wait for data: False + + +Operations +====================================== + + +Or +-------------------------------------- + +.. autopydantic_model:: Or + :inherited-members: BaseModel + + +Example +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.. + YamlModel: Or + +.. code-block:: yaml + + or: + - type: change filter + - heartbeat action: 60 + + +Sequence +-------------------------------------- + +.. autopydantic_model:: Sequence + :inherited-members: BaseModel + + +Example +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.. + YamlModel: Sequence + +.. code-block:: yaml + + sequence: + - factor: 0.1 + - offset: -50 + + + +Examples +====================================== +.. py:currentmodule:: sml2mqtt.config.device + +These are some examples for sml value configurations + +Energy consumption today +-------------------------------------- + +This will report the power consumption of today. +The first reported value every day will be 0 and then it will increase for every day. + +.. + YamlModel: SmlValueConfig + +.. code-block:: yaml + + obis: '0100010800ff' # Obis code for the energy meter + mqtt: + topic: energy_today # MQTT topic for the meter + operations: + - type: meter + start now: true # Start immediately + reset times: # Reset at midnight + - 00:00 + - round: 1 + - type: change filter # Only report on changes + - refresh action: 01:00 # ... but refresh every hour + + +Downsample current power +-------------------------------------- + +This will report a power value every max every 30s. +The reported value will be the weighted mean value of the last 30s. + +.. + YamlModel: SmlValueConfig + +.. code-block:: yaml + + obis: '0100100700ff' # Obis code for the energy meter + mqtt: + topic: power # MQTT topic for the meter + operations: + - type: mean interval # Calculate weighted mean over 30s + interval: 30 + wait for data: False + - throttle filter: 30 # Let a value pass every 30s + - round: 0 # Round the mean value to the full number + - type: delta filter # Only report when the value changes at least 10W or 5% + min: 10 + min %: 5 + - refresh action: 01:00 # ... but refresh every hour diff --git a/docs/requirements.txt b/docs/requirements.txt index 1dfe91c..02e630c 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,6 +1,6 @@ # Packages required to build the documentation -sphinx >= 6, < 7 -sphinx-autodoc-typehints >= 1.22, < 2 -sphinx_rtd_theme == 1.2.0 -sphinx-exec-code == 0.10 -autodoc_pydantic >= 1.8, < 1.9 +sphinx == 7.2.6 +sphinx-autodoc-typehints == 2.0.0 +sphinx_rtd_theme == 2.0.0 +sphinx-exec-code == 0.12 +autodoc_pydantic == 2.1.0 diff --git a/readme.md b/readme.md index 7152db7..a778969 100644 --- a/readme.md +++ b/readme.md @@ -6,18 +6,30 @@ [![Docker Image Version (latest by date)](https://img.shields.io/docker/v/spacemanspiff2007/sml2mqtt?label=docker)](https://hub.docker.com/r/spacemanspiff2007/sml2mqtt) [![Docker Pulls](https://img.shields.io/docker/pulls/spacemanspiff2007/sml2mqtt)](https://hub.docker.com/r/spacemanspiff2007/sml2mqtt) -_A simple yet flexible sml to mqtt bridge_ +_A simple yet extremely flexible sml to mqtt bridge_ sml2mqtt is a asyncio application that can read multiple sml (Smart Message Language) streams from energy meters and report the values through mqtt. +The meters can be read through serial ports or through http(s) (e.g. Tibber devices) +To read from the serial port an IR to USB reader for energy meter is required. # Documentation [The documentation can be found at here](https://sml2mqtt.readthedocs.io) # Changelog + +#### 3.0 (2024-04-24) + +**BREAKING CHANGE** + +- Almost complete rewrite, requires at least **Python 3.10** +- Extensive value processing which can be configured -> **Config file changed** +- Support for tibber pulse out of the box +- The ``analyze`` flag can also be set through an environment variable which makes it easier for docker users + #### 2.2 (2023-03-31) - Small config improvements diff --git a/requirements.txt b/requirements.txt index 2cbd607..db81569 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,10 @@ -r requirements_setup.txt # Testing -pytest >= 7.1, < 8 -pre-commit >= 2, < 3 -pytest-asyncio >= 0.19, < 0.20 +pytest == 8.1.1 +pre-commit == 3.7.0 +pytest-asyncio == 0.23.6 +aioresponses == 0.7.6 + +# Linter +ruff == 0.4.1 diff --git a/requirements_setup.txt b/requirements_setup.txt index 124c031..62b5477 100644 --- a/requirements_setup.txt +++ b/requirements_setup.txt @@ -1,5 +1,6 @@ -asyncio-mqtt == 0.16.1 +aiomqtt == 2.0.1 pyserial-asyncio == 0.6 -easyconfig == 0.2.8 -pydantic >= 1.10, <2.0 -smllib == 1.2 +easyconfig == 0.3.2 +pydantic == 2.7.0 +smllib == 1.4 +aiohttp == 3.9.5 diff --git a/setup.py b/setup.py index 6e8c29c..efc55e3 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,5 @@ -import typing +from __future__ import annotations + from pathlib import Path import setuptools # type: ignore @@ -6,14 +7,14 @@ # Load version number without importing HABApp def load_version() -> str: - version: typing.Dict[str, str] = {} + version: dict[str, str] = {} with open("src/sml2mqtt/__version__.py") as fp: exec(fp.read(), version) assert version['__version__'], version return version['__version__'] -def load_req() -> typing.List[str]: +def load_req() -> list[str]: req_file = Path(__file__).with_name('requirements_setup.txt') with req_file.open() as f: return f.readlines() @@ -36,12 +37,14 @@ def load_req() -> typing.List[str]: version=__version__, author="spaceman_spiff", # author_email="", - description="A sml (Smart Message Language) to MQTT bridge", + description="A sml (Smart Message Language) energy meter to MQTT bridge. " + "Can read from serial ports or http (e.g. Tibber Pulse).", keywords=[ 'mqtt', 'sml', 'Smart Message Language', - 'energy meter' + 'energy meter', + 'tibber' ], long_description=long_description, long_description_content_type="text/markdown", @@ -51,16 +54,16 @@ def load_req() -> typing.List[str]: }, packages=setuptools.find_packages(where='src', exclude=['tests*']), package_dir={'': 'src'}, - python_requires='>=3.8', + python_requires='>=3.10', install_requires=load_req(), classifiers=[ "Development Status :: 4 - Beta", "Framework :: AsyncIO", "License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)", "Operating System :: OS Independent", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3 :: Only", "Topic :: Home Automation" ], diff --git a/src/sml2mqtt/__args__.py b/src/sml2mqtt/__args__.py index 6ea0721..507e2d7 100644 --- a/src/sml2mqtt/__args__.py +++ b/src/sml2mqtt/__args__.py @@ -2,18 +2,20 @@ import os import sys from pathlib import Path -from typing import Final, Optional, Type +from typing import Final class CommandArgs: - config: Optional[Path] = None + config: Path | None = None analyze: bool = False CMD_ARGS: Final = CommandArgs -def get_command_line_args(args=None) -> Type[CommandArgs]: +def get_command_line_args(args=None) -> type[CommandArgs]: + + env_var_name = 'SML2MQTT_ANALYZE' parser = argparse.ArgumentParser(description='SML to MQTT bridge') parser.add_argument( @@ -25,23 +27,25 @@ def get_command_line_args(args=None) -> Type[CommandArgs]: parser.add_argument( '-a', '--analyze', - help='Process exactly one sml message, shows the values of the message and what will be reported', + help='Process exactly one sml message, shows the values of the message and what will be reported. ' + f'Can also be set by setting the environment variable "{env_var_name:s}" to an arbitrary value', action='store_true', default=False ) args = parser.parse_args(args) CMD_ARGS.config = find_config_folder(args.config) - CMD_ARGS.analyze = args.analyze + CMD_ARGS.analyze = args.analyze or bool(os.environ.get(env_var_name, '')) + return CMD_ARGS -def find_config_folder(config_file_str: Optional[str]) -> Path: +def find_config_folder(config_file_str: str | None) -> Path: check_path = [] if config_file_str is None: # Nothing is specified, we try to find the config automatically try: - working_dir = Path(os.getcwd()) + working_dir = Path.cwd() check_path.append(working_dir / 'sml2mqtt') check_path.append(working_dir.with_name('sml2mqtt')) check_path.append(working_dir.parent.with_name('sml2mqtt')) diff --git a/src/sml2mqtt/__init__.py b/src/sml2mqtt/__init__.py index 9573cb7..355e85c 100644 --- a/src/sml2mqtt/__init__.py +++ b/src/sml2mqtt/__init__.py @@ -2,14 +2,11 @@ from sml2mqtt.__args__ import CMD_ARGS from sml2mqtt.__version__ import __version__ + # isort: split from sml2mqtt.config import CONFIG # isort: split -from sml2mqtt import sml_value - # isort: split - -import sml2mqtt.device diff --git a/src/sml2mqtt/__log__.py b/src/sml2mqtt/__log__.py index 10f5df6..6c11b47 100644 --- a/src/sml2mqtt/__log__.py +++ b/src/sml2mqtt/__log__.py @@ -2,9 +2,11 @@ import sys from datetime import date, datetime from logging.handlers import RotatingFileHandler +from pathlib import Path import sml2mqtt + log = logging.getLogger('sml') @@ -20,11 +22,10 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.last_check: date = datetime.now().date() - def shouldRollover(self, record): # noqa: N802 - date = datetime.now().date() - if date == self.last_check: + def shouldRollover(self, record) -> int: + if (date_now := datetime.now().date()) == self.last_check: return 0 - self.last_check = date + self.last_check = date_now return super().shouldRollover(record) @@ -33,19 +34,25 @@ def setup_log(): # This is the longest logger name str chars = 0 - for device in sml2mqtt.CONFIG.ports: - chars = max(len(f'sml.device.{device.url}'), chars) + for device in sml2mqtt.CONFIG.inputs: + # Name of the longest logger, should be the device status + chars = max(len(get_logger(device.get_device_name()).getChild('status').name), chars) log_format = logging.Formatter("[{asctime:s}] [{name:" + str(chars) + "s}] {levelname:8s} | {message:s}", style='{') # File Handler - log_file = sml2mqtt.CONFIG.logging.file - if not log_file.is_absolute(): - log_file = sml2mqtt.CMD_ARGS.config.parent / log_file - log_file.resolve() - - handler = MidnightRotatingFileHandler( - str(log_file), maxBytes=1024 * 1024, backupCount=3, encoding='utf-8' - ) + file_path = sml2mqtt.CONFIG.logging.file + if file_path.lower() == 'stdout': + handler = logging.StreamHandler(sys.stdout) + else: + log_file = Path(file_path) + if not log_file.is_absolute(): + log_file = sml2mqtt.CMD_ARGS.config.parent / log_file + log_file.resolve() + + handler = MidnightRotatingFileHandler( + str(log_file), maxBytes=1024 * 1024, backupCount=3, encoding='utf-8' + ) + handler.setFormatter(log_format) handler.setLevel(logging.DEBUG) diff --git a/src/sml2mqtt/__main__.py b/src/sml2mqtt/__main__.py index e762728..d78ec04 100644 --- a/src/sml2mqtt/__main__.py +++ b/src/sml2mqtt/__main__.py @@ -1,57 +1,62 @@ import asyncio -import platform +import os import sys import traceback -import typing from sml2mqtt import mqtt from sml2mqtt.__args__ import CMD_ARGS, get_command_line_args from sml2mqtt.__log__ import log, setup_log -from sml2mqtt.__shutdown__ import get_return_code, shutdown, signal_handler_setup -from sml2mqtt.config import CONFIG -from sml2mqtt.device import Device +from sml2mqtt.config import CONFIG, cleanup_validation_errors +from sml2mqtt.const.task import wait_for_tasks +from sml2mqtt.runtime import do_shutdown_async, on_shutdown, signal_handler_setup +from sml2mqtt.sml_device import ALL_DEVICES, SmlDevice +from sml2mqtt.sml_source import create_source async def a_main(): - devices = [] + # Add possibility to stop program with Ctrl + c + signal_handler_setup() + + on_shutdown(ALL_DEVICES.cancel_and_wait, 'Stop devices') try: - if CMD_ARGS.analyze: + if analyze := CMD_ARGS.analyze: mqtt.patch_analyze() else: # initial mqtt connect - mqtt.start() + await mqtt.start() await mqtt.wait_for_connect(5) - # Create devices for port - for port_cfg in CONFIG.ports: - dev_mqtt = mqtt.BASE_TOPIC.create_child(port_cfg.url) - device = await Device.create(port_cfg, port_cfg.timeout, set(), dev_mqtt) - devices.append(device) + # Create device for each input + for input_cfg in CONFIG.inputs: + device = ALL_DEVICES.add_device(SmlDevice(input_cfg.get_device_name())) + device.set_source(await create_source(device, settings=input_cfg)) + device.watchdog.set_timeout(input_cfg.timeout) + if analyze: + device.frame_handler = device.analyze_frame - for device in devices: - device.start() + # Start all devices + await ALL_DEVICES.start() - except Exception as e: - shutdown(e) + except Exception: + await do_shutdown_async() - return await asyncio.gather(*devices, mqtt.wait_for_disconnect()) + # Keep tasks running + await wait_for_tasks() -def main() -> typing.Union[int, str]: - try: - CONFIG.load_config_file(get_command_line_args().config) - except Exception as e: - print(e) - return 7 - +def main() -> int | str: # This is needed to make async-mqtt work # see https://github.com/sbtinstruments/asyncio-mqtt - if platform.system() == 'Windows': + if sys.platform.lower() == "win32" or os.name.lower() == "nt": asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) - # Add possibility to stop program with Ctrl + c - signal_handler_setup() + # Load config + try: + CONFIG.load_config_file(get_command_line_args().config) + except Exception as e: + print(cleanup_validation_errors(str(e))) + return 2 try: setup_log() @@ -66,7 +71,7 @@ def main() -> typing.Union[int, str]: print(e) return str(e) - return get_return_code() + return 0 if __name__ == "__main__": diff --git a/src/sml2mqtt/__shutdown__.py b/src/sml2mqtt/__shutdown__.py deleted file mode 100644 index 216450c..0000000 --- a/src/sml2mqtt/__shutdown__.py +++ /dev/null @@ -1,112 +0,0 @@ -import signal -import traceback -from asyncio import create_task, Task -from typing import Dict, Optional, Type, Union - -import sml2mqtt.mqtt -from sml2mqtt.__log__ import log -from sml2mqtt.errors import AllDevicesFailedError, DeviceSetupFailedError, InitialMqttConnectionFailedError - -# ---------------------------------------------------------------------------------------------------------------------- -# Return code logic -# ---------------------------------------------------------------------------------------------------------------------- -_RETURN_CODE: Optional[int] = None - - -def set_return_code(code: int): - global _RETURN_CODE - if _RETURN_CODE is None: - _RETURN_CODE = code - else: - if _RETURN_CODE != code: - log.debug(f'Return code is already set to {_RETURN_CODE}, skip setting {code}!') - - -def get_return_code() -> int: - if _RETURN_CODE is None: - log.warning('No return code set!') - return 2 - return _RETURN_CODE - - -# ---------------------------------------------------------------------------------------------------------------------- -# Signal handlers so we can shutdown gracefully -# ---------------------------------------------------------------------------------------------------------------------- -def _signal_handler_shutdown(sig, frame): - set_return_code(0) - do_shutdown() - - -def signal_handler_setup(): - signal.signal(signal.SIGINT, _signal_handler_shutdown) - signal.signal(signal.SIGTERM, _signal_handler_shutdown) - - -# ---------------------------------------------------------------------------------------------------------------------- -# Actual shutdown logic -# ---------------------------------------------------------------------------------------------------------------------- -SHUTDOWN_TASK: Optional[Task] = None -SHUTDOWN_REQUESTED = False - - -def do_shutdown(): - global SHUTDOWN_TASK, SHUTDOWN_REQUESTED - - if SHUTDOWN_REQUESTED: - return None - - if SHUTDOWN_TASK is None: - SHUTDOWN_TASK = create_task(_shutdown_task()) - - SHUTDOWN_REQUESTED = True - - -async def _shutdown_task(): - global SHUTDOWN_TASK - - try: - print('Shutting down ...') - log.info('Shutting down ...') - - sml2mqtt.mqtt.cancel() - - # once all devices are stopped the main loop will exit - for device in sml2mqtt.device.sml_device.ALL_DEVICES.values(): - device.stop() - finally: - SHUTDOWN_TASK = None - - -# ---------------------------------------------------------------------------------------------------------------------- -# shutdown helper -# ---------------------------------------------------------------------------------------------------------------------- -def shutdown(e: Union[Exception, Type[Exception]]): - global SHUTDOWN_TASK - - ret_map: Dict[int, Type[Exception]] = { - 10: DeviceSetupFailedError, - 11: InitialMqttConnectionFailedError, - 20: AllDevicesFailedError - } - - log_traceback = True - - # get return code based on the error - for ret_code, cls in ret_map.items(): # noqa: B007 - if isinstance(e, cls): - log_traceback = False - break - - if e is cls: - log_traceback = False - break - else: - ret_code = 1 - - if log_traceback: - for line in traceback.format_exc().splitlines(): - log.error(line) - - set_return_code(ret_code) - - do_shutdown() diff --git a/src/sml2mqtt/__version__.py b/src/sml2mqtt/__version__.py index 2b9ccf1..f7d2e86 100644 --- a/src/sml2mqtt/__version__.py +++ b/src/sml2mqtt/__version__.py @@ -1 +1 @@ -__version__ = '2.2' +__version__ = '3.0' diff --git a/src/sml2mqtt/config/__init__.py b/src/sml2mqtt/config/__init__.py index bd1803d..927b0e4 100644 --- a/src/sml2mqtt/config/__init__.py +++ b/src/sml2mqtt/config/__init__.py @@ -1,2 +1,3 @@ from .config import CONFIG from .mqtt import OptionalMqttPublishConfig +from .operations import cleanup_validation_errors diff --git a/src/sml2mqtt/config/config.py b/src/sml2mqtt/config/config.py index 9aba0ec..44f508e 100644 --- a/src/sml2mqtt/config/config.py +++ b/src/sml2mqtt/config/config.py @@ -1,60 +1,20 @@ -from typing import Dict, List, Union - -import serial from easyconfig import AppBaseModel, BaseModel, create_app_config -from pydantic import constr, Field, StrictFloat, StrictInt, StrictStr, validator +from pydantic import Field -from .device import REPUBLISH_ALIAS, SmlDeviceConfig, SmlValueConfig +from .device import SmlDeviceConfig, SmlValueConfig +from .inputs import HttpSourceSettings, SerialSourceSettings from .logging import LoggingSettings from .mqtt import MqttConfig, OptionalMqttPublishConfig - - -class PortSettings(BaseModel): - url: constr(strip_whitespace=True, min_length=1, strict=True) = Field(..., description='Device path') - timeout: Union[int, float] = Field( - default=3, description='Seconds after which a timeout will be detected (default=3)') - - baudrate: int = Field(9600, in_file=False) - parity: str = Field('None', in_file=False) - stopbits: Union[StrictInt, StrictFloat] = Field(serial.STOPBITS_ONE, in_file=False, alias='stop bits') - bytesize: int = Field(serial.EIGHTBITS, in_file=False, alias='byte size') - - @validator('baudrate') - def _val_baudrate(cls, v): - if v not in serial.Serial.BAUDRATES: - raise ValueError(f'must be one of {list(serial.Serial.BAUDRATES)}') - return v - - @validator('parity') - def _val_parity(cls, v): - # Short name - if v in serial.PARITY_NAMES: - return v - - # Name -> Short name - parity_values = {_n: _v for _v, _n in serial.PARITY_NAMES.items()} - if v not in parity_values: - raise ValueError(f'must be one of {list(parity_values)}') - return parity_values[v] - - @validator('stopbits') - def _val_stopbits(cls, v): - if v not in serial.Serial.STOPBITS: - raise ValueError(f'must be one of {list(serial.Serial.STOPBITS)}') - return v - - @validator('bytesize') - def _val_bytesize(cls, v): - if v not in serial.Serial.BYTESIZES: - raise ValueError(f'must be one of {list(serial.Serial.BYTESIZES)}') - return v +from .types import LowerStr, ObisHex class GeneralSettings(BaseModel): - wh_in_kwh: bool = Field(True, description='Automatically convert Wh to kWh', alias='Wh in kWh') + wh_in_kwh: bool = Field( + True, description='Automatically convert Wh to kWh', alias='Wh in kWh' + ) republish_after: int = Field( 120, description='Republish automatically after this time (if no other filter configured)', - alias=REPUBLISH_ALIAS, + alias='republish after', ) report_blank_energy_meters: bool = Field( False, description='Report blank energy meters (where the value is 0kwh)', @@ -64,7 +24,7 @@ class GeneralSettings(BaseModel): False, description='Report the device id even though it does never change', alias='report device id', in_file=False ) - device_id_obis: List[StrictStr] = Field( + device_id_obis: list[ObisHex] = Field( # 0100000009ff (1-0:0.0.9*255) : Geräteeinzelidentifikation # 0100600100ff (1-0:96.1.0*255): Produktionsnummer ['0100000009ff', '0100600100ff'], @@ -77,31 +37,35 @@ class Settings(AppBaseModel): logging: LoggingSettings = Field(default_factory=LoggingSettings) mqtt: MqttConfig = Field(default_factory=MqttConfig) general: GeneralSettings = Field(default_factory=GeneralSettings) - ports: List[PortSettings] = [] - devices: Dict[str, SmlDeviceConfig] = Field({}, description='Device configuration by ID or url',) + inputs: list[HttpSourceSettings | SerialSourceSettings] = Field([], discriminator='type') + devices: dict[LowerStr, SmlDeviceConfig] = Field({}, description='Device configuration by ID or url',) def default_config() -> Settings: # File defaults - s = Settings( - ports=[PortSettings(url='COM1', timeout=3), PortSettings(url='/dev/ttyS0', timeout=3), ], + return Settings( + inputs=[SerialSourceSettings(type='serial', url='COM1', timeout=6), + SerialSourceSettings(type='serial', url='/dev/ttyS0', timeout=6), ], devices={ - 'DEVICE_ID_HEX': SmlDeviceConfig( + 'device_id_hex': SmlDeviceConfig( mqtt=OptionalMqttPublishConfig(topic='DEVICE_BASE_TOPIC'), status=OptionalMqttPublishConfig(topic='status'), - skip=['OBIS'], - values={ - 'OBIS': SmlValueConfig( + skip={'00112233445566'}, + values=[ + SmlValueConfig( + obis='00112233445566', mqtt=OptionalMqttPublishConfig(topic='OBIS'), - workarounds=[{'negative on energy meter status': True}], - transformations=[{'factor': 3}, {'offset': 100}, {'round': 2}], - filters=[{'diff': 10}, {'perc': 10}, {'every': 120}], + operations=[ + {'negative on energy meter status': True}, + {'factor': 3}, {'offset': 100}, {'round': 2}, + {'type': 'change filter'}, + {'refresh action': 600} + ] ) - } + ] ) } ) - return s CONFIG: Settings = create_app_config(Settings(), default_config) diff --git a/src/sml2mqtt/config/device.py b/src/sml2mqtt/config/device.py index 4eb9fe3..a3c911b 100644 --- a/src/sml2mqtt/config/device.py +++ b/src/sml2mqtt/config/device.py @@ -1,70 +1,38 @@ -from enum import Enum -from typing import Dict, List, Optional, Set, Union - from easyconfig import BaseModel -from pydantic import Field, StrictBool, StrictFloat, StrictInt, StrictStr, validator +from pydantic import Field from .mqtt import OptionalMqttPublishConfig - -REPUBLISH_ALIAS = 'republish after' - - -class WorkaroundOptionEnum(str, Enum): - negative_on_energy_meter_status = 'negative on energy meter status' - - -class TransformOptionEnum(str, Enum): - factor = 'factor' #: Use the value as a factor - offset = 'offset' #: Use the value as an offset - round = 'round' #: Round the result to the digits - - -class FilterOptionEnum(str, Enum): - diff = 'diff' #: Report when difference is greater equal than the value - perc = 'perc' #: Report when percentual change is greater equal the value - every = 'every' #: Report every x seconds - - -TYPE_SML_VALUE_WORKAROUND_CFG = \ - Optional[List[Dict[WorkaroundOptionEnum, Union[StrictBool, StrictInt, StrictFloat, StrictStr]]]] -TYPE_SML_VALUE_TRANSFORM_CFG = Optional[List[Dict[TransformOptionEnum, Union[StrictInt, StrictFloat]]]] -TYPE_SML_VALUE_FILTER_CFG = Optional[List[Dict[FilterOptionEnum, Union[StrictInt, StrictFloat]]]] +from .operations import OperationsListType +from .types import ObisHex class SmlValueConfig(BaseModel): - mqtt: OptionalMqttPublishConfig = Field(None, description='Mqtt config for this entry (optional)') - - workarounds: TYPE_SML_VALUE_WORKAROUND_CFG = Field( - None, description='Workarounds for the value (optional)') - transformations: TYPE_SML_VALUE_TRANSFORM_CFG = Field( - None, description='Mathematical transformations for the value (optional)') - filters: TYPE_SML_VALUE_FILTER_CFG = Field( - None, description='Refresh options for the value (optional)') - - @validator('workarounds', 'transformations', 'filters') - def len_1(cls, v): - if v is None: - return None + obis: ObisHex = Field(description='Obis code for this value') + mqtt: OptionalMqttPublishConfig | None = Field( + default=None, + description='Mqtt config for this value (optional)' + ) - for entry in v: - if len(entry) != 1: - raise ValueError(f'Only one entry allowed! Got {len(entry)}: {", ".join(entry.keys())}') - return v + operations: OperationsListType = Field( + default=[], + alias='operations', + description='A sequence of operations that will be evaluated one after another.\n' + 'If one operation blocks this will return nothing.' + ) class SmlDeviceConfig(BaseModel): """Configuration for a sml device""" - mqtt: Optional[OptionalMqttPublishConfig] = Field( - default=None, description='Optional MQTT configuration for this meter.') + mqtt: OptionalMqttPublishConfig | None = Field(None, description='Optional MQTT configuration for this meter.') - status: Optional[OptionalMqttPublishConfig] = Field( + status: OptionalMqttPublishConfig = Field( default=OptionalMqttPublishConfig(topic='status'), description='Optional MQTT status topic configuration for this meter' ) - skip: Optional[Set[StrictStr]] = Field( - default=None, description='OBIS codes (HEX) of values that will not be published (optional)') + skip: set[ObisHex] = Field( + default_factory=set, description='OBIS codes (HEX) of values that will not be published (optional)') - values: Dict[StrictStr, SmlValueConfig] = Field( - default={}, description='Special configurations for each of the values (optional)') + values: list[SmlValueConfig] = Field( + default=[], description='Configurations for each of the values (optional)') diff --git a/src/sml2mqtt/config/inputs.py b/src/sml2mqtt/config/inputs.py new file mode 100644 index 0000000..8091fa5 --- /dev/null +++ b/src/sml2mqtt/config/inputs.py @@ -0,0 +1,114 @@ +from typing import Literal + +import serial +from aiohttp import ClientTimeout +from easyconfig import BaseModel +from pydantic import ( + AnyHttpUrl, + Field, + StrictFloat, + StrictInt, + constr, + field_validator, + model_validator, +) +from typing_extensions import override + +from sml2mqtt.config.types import log + + +class SmlSourceSettingsBase(BaseModel): + def get_device_name(self) -> str: + raise NotImplementedError() + + +class SerialSourceSettings(SmlSourceSettingsBase): + type: Literal['serial'] + + url: constr(strip_whitespace=True, min_length=1, strict=True) = Field(..., description='Device path') + timeout: StrictInt | StrictFloat = Field( + default=6, description='Seconds after which a timeout will be detected (default=6)') + + baudrate: int = Field(9600, in_file=False) + parity: str = Field('None', in_file=False) + stopbits: StrictInt | StrictFloat = Field(serial.STOPBITS_ONE, in_file=False, alias='stop bits') + bytesize: int = Field(serial.EIGHTBITS, in_file=False, alias='byte size') + + @field_validator('baudrate') + def _val_baudrate(cls, v): + if v not in serial.Serial.BAUDRATES: + msg = f'must be one of {list(serial.Serial.BAUDRATES)}' + raise ValueError(msg) + return v + + @field_validator('parity') + def _val_parity(cls, v): + # Short name + if v in serial.PARITY_NAMES: + return v + + # Name -> Short name + parity_values = {_n: _v for _v, _n in serial.PARITY_NAMES.items()} + if v not in parity_values: + msg = f'must be one of {list(parity_values)}' + raise ValueError(msg) + return parity_values[v] + + @field_validator('stopbits') + def _val_stopbits(cls, v): + if v not in serial.Serial.STOPBITS: + msg = f'must be one of {list(serial.Serial.STOPBITS)}' + raise ValueError(msg) + return v + + @field_validator('bytesize') + def _val_bytesize(cls, v): + if v not in serial.Serial.BYTESIZES: + msg = f'must be one of {list(serial.Serial.BYTESIZES)}' + raise ValueError(msg) + return v + + @override + def get_device_name(self) -> str: + return self.url.split("/")[-1] + + +class HttpSourceSettings(SmlSourceSettingsBase): + type: Literal['http'] + + url: AnyHttpUrl = Field(..., description='Url') + timeout: StrictInt | StrictFloat = Field( + default=6, description='Seconds after which a timeout will be detected (default=6)') + + interval: StrictInt | StrictFloat = Field(default=2, description='Delay between requests', ge=0.1) + user: str = Field(default='', description='User (if needed)') + password: str = Field(default='', description='Password (if needed)') + + request_timeout: StrictInt | StrictFloat | None = Field( + default=None, alias='request timeout', description='Dedicated timeout for the http request', + in_file=False + ) + + @override + def get_device_name(self) -> str: + return self.url.host + + @model_validator(mode='after') + def check_timeout_gt_interval(self): + if self.interval * 2 > self.timeout: + msg = 'Timeout must be greater equal than 2 * interval' + raise ValueError(msg) + + # Timeout is interval, and we automatically retry 3 times before we fail + if self.interval * 3 > self.timeout: + log.warning(f'The recommendation for timeout is at least 3 * interval ' + f'({self.interval * 3:.0f})! Currently: {self.timeout}') + + return self + + def get_request_timeout(self) -> ClientTimeout: + value = self.interval if self.request_timeout is None else self.request_timeout + if value is None: + msg = 'No value for ClientTimeout' + raise ValueError(msg) + return ClientTimeout(total=value) diff --git a/src/sml2mqtt/config/logging.py b/src/sml2mqtt/config/logging.py index 38f74e0..da3e308 100644 --- a/src/sml2mqtt/config/logging.py +++ b/src/sml2mqtt/config/logging.py @@ -1,21 +1,18 @@ import logging -from pathlib import Path from easyconfig import BaseModel -from pydantic import Extra, Field, validator +from pydantic import Extra, Field, field_validator class LoggingSettings(BaseModel): level: str = Field('INFO', description='Log level') - file: Path = Field('sml2mqtt.log', description='Log file path (absolute or relative to config file)') + file: str = Field('sml2mqtt.log', description='Log file path (absolute or relative to config file) or "stdout"') - class Config: - extra = Extra.forbid - - @validator('level') + @field_validator('level') def validate_logging(cls, value): if value not in logging._nameToLevel: - raise ValueError(f'Level must be one of {", ".join(logging._nameToLevel)}') + msg = f'Level must be one of {", ".join(logging._nameToLevel)}' + raise ValueError(msg) return value def set_log_level(self) -> int: diff --git a/src/sml2mqtt/config/mqtt.py b/src/sml2mqtt/config/mqtt.py index bc6536d..5f01768 100644 --- a/src/sml2mqtt/config/mqtt.py +++ b/src/sml2mqtt/config/mqtt.py @@ -1,59 +1,67 @@ +import random +import string + from easyconfig import BaseModel -from pydantic import conint, constr, Field, StrictBool, validator +from pydantic import Field, StrictBool, field_validator, model_validator -QOS = conint(ge=0, le=2) -TOPIC_STR = constr(strip_whitespace=True, min_length=1) -STRIPPED_STR = constr(strip_whitespace=True) +from sml2mqtt.config.mqtt_tls import MqttTlsOptions +from sml2mqtt.config.types import MqttQosInt, MqttTopicStr, StrippedStr class MqttDefaultPublishConfig(BaseModel): - qos: QOS = Field( - 0, description='Default value for QOS if no other QOS value in the config entry is set') + qos: MqttQosInt = Field( + 0, description='Default value for QoS if no other QoS value in the config entry is set') retain: StrictBool = Field( False, description='Default value for retain if no other retain value in the config entry is set') class OptionalMqttPublishConfig(BaseModel): - topic: TOPIC_STR = Field(None, description='Topic fragment for building this topic with the parent topic') - full_topic: TOPIC_STR = Field( + topic: MqttTopicStr | None = Field( + None, description='Topic fragment for building this topic with the parent topic') + full_topic: MqttTopicStr | None = Field( None, alias='full topic', description='Full topic - will ignore the parent topic parts') - qos: QOS = Field(None, description='QoS for publishing this value (if set - otherwise use parent)') - retain: StrictBool = Field(None, description='Retain for publishing this value (if set - otherwise use parent)') + qos: MqttQosInt | None = Field( + None, description='QoS for publishing this value (if set - otherwise use parent)') + retain: StrictBool | None = Field( + None, description='Retain for publishing this value (if set - otherwise use parent)') - @validator('topic', 'full_topic') + @field_validator('topic', 'full_topic') def validate_topic(cls, value): if value is None: return None if value.endswith('/'): - raise ValueError('Topic must not end with "/"') + msg = 'Topic must not end with "/"' + raise ValueError(msg) if value.startswith('/'): - raise ValueError('Topic must not start with "/"') + msg = 'Topic must not start with "/"' + raise ValueError(msg) return value - @validator('full_topic') - def check_full_or_partial(cls, v, values): - if v is None: - return None - - if values.get('topic') is not None: - raise ValueError('Topic and full_topic can not be used at the same time!') - return v + @model_validator(mode='after') + def check_full_or_partial(self): + if self.topic is not None and self.full_topic is not None: + msg = 'Topic and full_topic can not be used at the same time!' + raise ValueError(msg) + return self class MqttConnection(BaseModel): - client_id: STRIPPED_STR = Field('sml2mqtt', alias='client id') - host: STRIPPED_STR = 'localhost' - port: conint(gt=0) = 1883 - user: STRIPPED_STR = '' - password: STRIPPED_STR = '' - tls: StrictBool = False - tls_insecure: StrictBool = Field(False, alias='tls insecure') + identifier: StrippedStr = Field('sml2mqtt-' + ''.join(random.choices(string.ascii_letters, k=13)),) + host: StrippedStr = 'localhost' + port: int = Field(1883, ge=0) + user: StrippedStr = '' + password: StrippedStr = '' + + tls: MqttTlsOptions | None = Field(None) class MqttConfig(BaseModel): - connection: MqttConnection = Field(default_factory=MqttConnection) - topic: TOPIC_STR = Field('sml2mqtt', alias='topic prefix') - defaults: MqttDefaultPublishConfig = Field(default_factory=MqttDefaultPublishConfig) + connection: MqttConnection = Field( + default_factory=MqttConnection) + topic: StrippedStr = Field( + 'sml2mqtt', alias='topic prefix', description='Prefix for all topics. Set to empty string to disable') + defaults: MqttDefaultPublishConfig = Field( + default_factory=MqttDefaultPublishConfig) last_will: OptionalMqttPublishConfig = Field( default_factory=lambda: OptionalMqttPublishConfig(topic='status'), alias='last will') diff --git a/src/sml2mqtt/config/mqtt_tls.py b/src/sml2mqtt/config/mqtt_tls.py new file mode 100644 index 0000000..81dcbde --- /dev/null +++ b/src/sml2mqtt/config/mqtt_tls.py @@ -0,0 +1,87 @@ +import ssl +from logging import Logger +from pathlib import Path +from typing import Any, Literal + +from aiomqtt import TLSParameters +from easyconfig import BaseModel +from pydantic import Field, field_validator + + +def _get_ssl_consts(prefix: str) -> dict[str, int]: + return {name.removeprefix(prefix): getattr(ssl, name) for name in dir(ssl) if name.startswith(prefix)} + + +def get_ssl_verify_values() -> dict[str, int]: + return _get_ssl_consts('CERT_') + + +def get_ssl_version_values() -> dict[str, int]: + return _get_ssl_consts('PROTOCOL_') + + +class MqttTlsOptions(BaseModel): + insecure: bool | None = Field( + None, description='Enable/disable server hostname verification when using SSL/TLS.' + ) + + ca_certs: str | None = Field( + None, alias='ca certificates', + description='Path to Certificate Authority (CA) certificate file in PEM or DER format' + ) + + cert_file: str | None = Field( + None, alias='cert file', description='Path to PEM encoded client certificate file' + ) + key_file: str | None = Field( + None, alias='key file', description='Path to PEM encoded private keys file' + ) + file_password: str | None = Field( + None, alias='file password', description='Password to encrypt the cert file or the key file if needed' + ) + + cert_reqs: Literal[tuple(get_ssl_verify_values())] | None = Field( + None, alias='certificate requirement', + description='Certificate requirement that the client imposes on the broker.' + ) + tls_version: Literal[tuple(get_ssl_version_values())] | None = Field( + None, alias='tls version', description='The version of the SSL/TLS protocol to be used.' + ) + ciphers: str | None = Field( + None, description='Which encryption ciphers are allowable for the connection' + ) + + @field_validator('ca_certs', 'cert_file', 'key_file') + @classmethod + def _ensure_file_exists(cls, v: str | None): + if v is None: + return None + + if not isinstance(v, str): + raise TypeError() + + if not Path(v).is_file(): + raise FileNotFoundError(v) + + return v + + def get_client_kwargs(self, log: Logger) -> dict[str, Any]: + cert_reqs = get_ssl_verify_values()[self.cert_reqs] if self.cert_reqs is not None else None + tls_version = get_ssl_version_values()[self.tls_version] if self.tls_version is not None else None + + if self.insecure: + log.warning('Verification of server hostname in server certificate disabled! ' + 'Use this only for testing, not for a real system!') + + return { + 'tls_insecure': self.insecure, + 'tls_params': TLSParameters( + ca_certs=self.ca_certs, + certfile=self.cert_file, + keyfile=self.key_file, + cert_reqs=cert_reqs, + tls_version=tls_version, + ciphers=self.ciphers, + keyfile_password=self.file_password + ) + } diff --git a/src/sml2mqtt/config/operations.py b/src/sml2mqtt/config/operations.py new file mode 100644 index 0000000..44f01ef --- /dev/null +++ b/src/sml2mqtt/config/operations.py @@ -0,0 +1,324 @@ +from __future__ import annotations + +from datetime import date, time, timedelta +from enum import Enum +from typing import Annotated, Any, Final, Literal, TypeAlias, TypedDict, Union, final +from typing import get_args as _get_args + +from annotated_types import Len +from easyconfig import BaseModel +from pydantic import Discriminator, Field, StrictBool, StrictFloat, StrictInt, Tag, model_validator + +from sml2mqtt.const import DateTimeFinder, DurationType, TimeSeries + +from .types import Number, ObisHex, PercentStr # noqa: TCH001 + + +class EmptyKwargs(TypedDict): + pass + + +# ------------------------------------------------------------------------------------------------- +# Filters +# ------------------------------------------------------------------------------------------------- +class OnChangeFilter(BaseModel): + """A filter which lets the value only pass when it's different from the value that was passed the last time""" + type: Literal['change filter'] = Field(description='Filter which passes only changes') + + @final + def get_kwargs_on_change(self) -> EmptyKwargs: + return {} + + +class RangeFilter(BaseModel): + """Filters or limits to values that are in a certain range + + """ + type: Literal['range filter'] + min_value: float | None = Field(None, alias='min', description='minimum value that will pass') + max_value: float | None = Field(None, alias='max', description='maximum value that will pass') + limit_values: bool = Field( + False, alias='limit', description='Instead of ignoring the values they will be limited to min/max' + ) + + @model_validator(mode='after') + def _check_set(self) -> RangeFilter: + if self.min_value is None and self.max_value is None: + msg = 'Neither min or max are set!' + raise ValueError(msg) + return self + + +class DeltaFilter(BaseModel): + """A filter which lets the value only pass if the incoming value is different enough from value that was passed the + last time. The delta can an absolute value or as a percentage. + If multiple deltas are specified they are all checked. + """ + + type: Literal['delta filter'] + + min_value: StrictInt | StrictFloat | None = Field(None, alias='min') + min_percent: StrictInt | StrictFloat | None = Field(None, alias='min %') + + @model_validator(mode='after') + def _check_set(self) -> DeltaFilter: + if self.min_value is None and self.min_percent is None: + msg = 'Neither min or min % are set!' + raise ValueError(msg) + return self + + +class ThrottleFilter(BaseModel): + """Filter which only lets one value pass in the defined period. If the last passed value is not at least + ``period`` old any new value will not be forwarded. + """ + period: DurationType = Field(alias='throttle filter', description='Throttle period') + + +# ------------------------------------------------------------------------------------------------- +# Actions +# ------------------------------------------------------------------------------------------------- +class RefreshAction(BaseModel): + """Action which lets every value pass. When no value is received (e.g. because an earlier filter blocks) + this action will produce the last received value every interval. + """ + every: DurationType = Field(alias='refresh action', description='Refresh interval') + + +class HeartbeatAction(BaseModel): + """Action which lets a value pass periodically every specified interval. + When no value is received (e.g. because an earlier filter blocks) + this action will produce the last received value every interval. + """ + + every: DurationType = Field( + alias='heartbeat action', + description='Interval' + ) + +# ------------------------------------------------------------------------------------------------- +# Math +# ------------------------------------------------------------------------------------------------- +class Factor(BaseModel): + factor: Number = Field(description='Factor with which the value gets multiplied') + + +class Offset(BaseModel): + offset: Number = Field(description='Offset that gets added on the value') + + +class Round(BaseModel): + digits: int = Field(ge=0, le=6, alias='round', description='Round to the specified digits') + + +# ------------------------------------------------------------------------------------------------- +# Workarounds +# ------------------------------------------------------------------------------------------------- +class NegativeOnEnergyMeterWorkaround(BaseModel): + """Make value negative based on an energy meter status.""" + + enabled_or_obis: StrictBool | ObisHex = Field( + alias='negative on energy meter status', + description='Set to "true" to enable or to "false" to disable workaround. ' + 'If the default obis code for the energy meter is wrong set ' + 'to the appropriate meter obis code instead' + ) + + +# ------------------------------------------------------------------------------------------------- +# Operations +# ------------------------------------------------------------------------------------------------- +class Or(BaseModel): + """A sequence of operations that will be evaluated one after another. The first value that gets returned by an + operation will be used. + """ + operations: OperationsListType = Field(alias='or') + + +class Sequence(BaseModel): + """A sequence of operations that will be evaluated one after another. + If one operation blocks this will return nothing. + """ + operations: OperationsListType = Field(alias='sequence') + + +def generate_day_names() -> dict[str, int]: + # names of weekdays in local language + day_names: dict[str, int] = {date(2001, 1, i).strftime('%A'): i for i in range(1, 8)} + day_names.update({date(2001, 1, i).strftime('%A')[:3]: i for i in range(1, 8)}) + + # abbreviations in German and English + day_names.update({"Mo": 1, "Di": 2, "Mi": 3, "Do": 4, "Fr": 5, "Sa": 6, "So": 7}) + day_names.update({"Mon": 1, "Tue": 2, "Wed": 3, "Thu": 4, "Fri": 5, "Sat": 6, "Sun": 7}) + return {k.lower(): v for k, v in day_names.items()} + + +DayOfWeekStr = Enum('DayOfWeekStr', {k: k for k in generate_day_names()}, type=str) +DayOfMonth = Annotated[int, Field(ge=1, le=31, strict=True)] + + +# ------------------------------------------------------------------------------------------------- +# DateTime +# ------------------------------------------------------------------------------------------------- +class HasDateTimeFields(BaseModel): + + start_now: bool = Field( + alias='start now', description='Immediately start instead of starting after the next reset' + ) + reset_times: list[time] = Field( + default=[], alias='reset times', description='Time(s) of day when a reset will occur', + ) + reset_days: list[DayOfMonth | DayOfWeekStr] = Field( + default=[], alias='reset days', description='Days of month or weekdays where the time(s) will be checked' + ) + + @final + def get_kwargs_dt_fields(self) -> DateTimeBoundKwargs: + + names = generate_day_names() + dows = [names[n] for n in self.reset_days if not isinstance(n, int)] + days = [n for n in self.reset_days if isinstance(n, int)] + + finder = DateTimeFinder() + for t in self.reset_times: + finder.add_time(t) + for dow in dows: + finder.add_dow(dow) + for day in days: + finder.add_day(day) + + return { + 'dt_finder': finder, + 'start_now': self.start_now + } + + +class DateTimeBoundKwargs(TypedDict): + start_now: bool + dt_finder: DateTimeFinder + + +class VirtualMeter(HasDateTimeFields): + """A virtual meter. It will output the difference from the last reset""" + type: Literal['meter'] + + +class MaxValue(HasDateTimeFields): + """Maximum value since last reset""" + type: Literal['max value'] + + +class MinValue(HasDateTimeFields): + """Minimum value since last reset""" + type: Literal['min value'] + + +# ------------------------------------------------------------------------------------------------- +# TimeSeries +# ------------------------------------------------------------------------------------------------- +class TimeSeriesKwargs(TypedDict): + time_series: TimeSeries + reset_after_value: bool + + +class HasIntervalFields(BaseModel): + + interval: timedelta = Field( + description='Interval duration' + ) + + wait_for_data: bool = Field( + alias='wait for data', description='Only produce a value when data for the whole interval is available' + ) + + reset_after_value: bool = Field( + False, alias='reset after value', description='Clear all data as soon as a value has been produced' + ) + + @final + def get_kwargs_interval_fields(self) -> TimeSeriesKwargs: + return { + 'time_series': TimeSeries(self.interval, wait_for_data=self.wait_for_data), + 'reset_after_value': self.reset_after_value + } + + +class MaxOfInterval(HasIntervalFields): + """Maximum value in a sliding interval""" + type: Literal['max interval'] + + +class MinOfInterval(HasIntervalFields): + """Minimum value in a sliding interval""" + type: Literal['min interval'] + + +class MeanOfInterval(HasIntervalFields): + """Weighted mean in a sliding interval""" + type: Literal['mean interval'] + + +# ------------------------------------------------------------------------------------------------- + +OperationsModels = ( + OnChangeFilter, DeltaFilter, HeartbeatAction, RangeFilter, + RefreshAction, ThrottleFilter, + Factor, Offset, Round, + NegativeOnEnergyMeterWorkaround, + Or, Sequence, + VirtualMeter, MaxValue, MinValue, + MaxOfInterval, MinOfInterval, MeanOfInterval, +) + +# noinspection PyTypeHints +OperationsType: TypeAlias = Union[tuple(Annotated[o, Tag(o.__name__)] for o in OperationsModels)] # noqa: UP007 + + +MODEL_FIELD_MAP: Final[dict[str, frozenset[str]]] = { + _m.__name__: frozenset( + _f.alias if _f.alias is not None else _n for _n, _f in _m.model_fields.items() if _f.exclude is not True + ) + for _m in OperationsModels +} + +MODEL_TYPE_MAP: Final[dict[str, str]] = { + _get_args(_f.annotation)[0]: _m.__name__ + for _m in OperationsModels for _n, _f in _m.model_fields.items() if _n == 'type' +} + + +def check_allowed_keys(obj: Any): + if isinstance(obj, dict): + type = obj.get('type') # noqa: A001 + keys = set(obj) + else: + type = getattr(obj, 'type', None) # noqa: A001 + keys = set(obj.model_fields) + + # we have a type field + if type is not None: + return MODEL_TYPE_MAP.get(type) + + # let's see if we have a 100% match + for name, fields in MODEL_FIELD_MAP.items(): + if keys == fields: + return name + return None + + +OperationsTypeAnnotated: TypeAlias = Annotated[ + OperationsType, + Discriminator( + check_allowed_keys, + custom_error_type='invalid_key_names', + custom_error_message='Invalid key names', + custom_error_context={'discriminator': 'check_allowed_keys'} + ) +] + +OperationsListType = Annotated[list[OperationsTypeAnnotated], Len(min_length=1)] + + +def cleanup_validation_errors(msg: str) -> str: + # In the ValidationError there is the Model and the field, but the user should only be concerned by the field name + return msg.replace('Or.or', 'or').replace('Sequence.sequence', 'sequence') diff --git a/src/sml2mqtt/config/types.py b/src/sml2mqtt/config/types.py new file mode 100644 index 0000000..b6e6459 --- /dev/null +++ b/src/sml2mqtt/config/types.py @@ -0,0 +1,29 @@ +from typing import Annotated, Literal, TypeAlias + +from pydantic import Strict, StrictFloat, StrictInt, StringConstraints + +from sml2mqtt.__log__ import get_logger + + +log = get_logger('config') + + +ObisHex = Annotated[ + str, + StringConstraints(to_lower=True, strip_whitespace=True, pattern=r'[0-9a-fA-F]{12}', strict=True) +] + +LowerStr = Annotated[ + str, + StringConstraints(to_lower=True, strip_whitespace=True, strict=True) +] + + +Number: TypeAlias = StrictInt | StrictFloat + +PercentStr = Annotated[str, Strict(), StringConstraints(strip_whitespace=True, pattern=r'^\d+\.?\d*\s*%$')] + +StrippedStr = Annotated[str, Strict(), StringConstraints(strip_whitespace=True)] + +MqttTopicStr = Annotated[str, Strict(), StringConstraints(strip_whitespace=True, min_length=1)] +MqttQosInt: TypeAlias = Literal[0, 1, 2] diff --git a/src/sml2mqtt/const/__init__.py b/src/sml2mqtt/const/__init__.py new file mode 100644 index 0000000..273eb67 --- /dev/null +++ b/src/sml2mqtt/const/__init__.py @@ -0,0 +1,5 @@ +from .date_time_finder import DateTimeFinder, get_now +from .protocols import DeviceProto, SourceProto +from .sml_helpers import EnhancedSmlFrame, SmlFrameValues +from .task import DeviceTask, Task, create_task +from .time_series import DurationType, TimeSeries, get_duration diff --git a/src/sml2mqtt/const/date_time_finder.py b/src/sml2mqtt/const/date_time_finder.py new file mode 100644 index 0000000..5304f69 --- /dev/null +++ b/src/sml2mqtt/const/date_time_finder.py @@ -0,0 +1,61 @@ +from datetime import datetime, timedelta +from datetime import time as dt_time + + +def get_now(): + return datetime.now() + + +class DateTimeFinder: + def __init__(self) -> None: + self.times: tuple[dt_time, ...] = () + self.dows: tuple[int, ...] = () + self.days: tuple[int, ...] = () + + self.enabled: bool = False + + @property + def condition_count(self) -> int: + return len(self.times) + len(self.dows) + len(self.days) + + def add_time(self, time: dt_time): + if not isinstance(time, dt_time): + raise TypeError() + self.times = (*self.times, time) + self.enabled = True + return self + + def add_dow(self, dow: int): + if not isinstance(dow, int): + raise TypeError() + if dow < 1 or dow > 7 or dow in self.dows: + raise ValueError() + self.dows = (*self.dows, dow) + return self + + def add_day(self, day: int): + if not isinstance(day, int): + raise TypeError() + if day < 1 or day > 31 or day in self.days: + raise ValueError() + self.days = (*self.days, day) + return self + + def get_first_reset(self, start_now: bool): + return get_now() if start_now or not self.enabled else self.calc_next() + + def calc_next(self, now: datetime | None = None) -> datetime: + if now is None: + now = get_now() + + next_dt = now + while True: + date = next_dt.date() + for time in self.times: + if (new := datetime.combine(date, time)) > now: + return new + + while True: + next_dt += timedelta(days=1) + if not self.dows and not self.days or next_dt.isoweekday() in self.dows or next_dt.day in self.days: + break diff --git a/src/sml2mqtt/const/protocols.py b/src/sml2mqtt/const/protocols.py new file mode 100644 index 0000000..a8053f9 --- /dev/null +++ b/src/sml2mqtt/const/protocols.py @@ -0,0 +1,26 @@ +from __future__ import annotations + +from typing_extensions import Protocol + + +class DeviceProto(Protocol): + @property + def name(self) -> str: + ... + + def on_source_data(self, data: bytes): + ... + + def on_source_failed(self, reason: str): + ... + + def on_error(self, e: Exception, *, show_traceback: bool = True): + ... + + +class SourceProto(Protocol): + def start(self): + ... + + def cancel_and_wait(self): + ... diff --git a/src/sml2mqtt/const/sml_helpers.py b/src/sml2mqtt/const/sml_helpers.py new file mode 100644 index 0000000..2c37fdc --- /dev/null +++ b/src/sml2mqtt/const/sml_helpers.py @@ -0,0 +1,80 @@ +from __future__ import annotations + +from binascii import b2a_hex +from time import monotonic +from typing import TYPE_CHECKING, Any, Final + +from smllib import SmlFrame + + +if TYPE_CHECKING: + from collections.abc import Generator, Iterable + from logging import Logger + + from smllib.sml import SmlListEntry + + +class EnhancedSmlFrame(SmlFrame): + + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + + self.timestamp: Final = monotonic() + + def get_frame_str(self): + yield 'Received Frame' + yield f' -> {b2a_hex(self.buffer)}' + + def get_analyze_str(self) -> Generator[str, None, None]: + yield '' + yield from self.get_frame_str() + yield '' + for obj in self.parse_frame(): + yield from obj.format_msg().splitlines() + yield '' + + def get_frame_values(self, log: Logger) -> SmlFrameValues: + + # try shortcut, if that fails try parsing the whole frame + try: + sml_objs = self.get_obis() # type: list[SmlListEntry] + except Exception: + log.info('get_obis failed - try parsing frame') + + sml_objs = [] # type: list[SmlListEntry] + for msg in self.parse_frame(): + for val in getattr(msg.message_body, 'val_list', []): + sml_objs.append(val) + + return SmlFrameValues.create(self.timestamp, sml_objs) + + +class SmlFrameValues: + @classmethod + def create(cls, timestamp: float, values: Iterable[SmlListEntry]): + c = cls(timestamp) + for value in values: + c.values[value.obis] = value + return c + + def __init__(self, timestamp: float): + self.timestamp: Final = timestamp + self.values: dict[str, SmlListEntry] = {} + + def __getattr__(self, item: str) -> SmlListEntry: + return self.values[item] + + def __len__(self): + return len(self.values) + + def get_value(self, obis: str) -> SmlListEntry | None: + return self.values.get(obis) + + def obis_ids(self) -> frozenset[str]: + return frozenset(self.values) + + def items(self, skip: set[str]) -> Generator[tuple[str, SmlListEntry], Any, None]: + for obis_id, value in self.values.items(): + if obis_id in skip: + continue + yield obis_id, value diff --git a/src/sml2mqtt/const/task.py b/src/sml2mqtt/const/task.py new file mode 100644 index 0000000..27ad6e6 --- /dev/null +++ b/src/sml2mqtt/const/task.py @@ -0,0 +1,113 @@ +from __future__ import annotations + +import logging +import traceback +from asyncio import CancelledError, Event, current_task +from asyncio import Task as asyncio_Task +from asyncio import create_task as asyncio_create_task +from typing import TYPE_CHECKING, Final + + +if TYPE_CHECKING: + from collections.abc import Awaitable, Callable, Coroutine + + from sml2mqtt.const.protocols import DeviceProto + + +TASKS: Final[set[asyncio_Task]] = set() + +log = logging.getLogger('sml.tasks') + + +def create_task(coro: Coroutine, *, name: str | None = None): + task = asyncio_create_task(coro, name=name) + + TASKS.add(task) + task.add_done_callback(TASKS.discard) + return task + + +async def wait_for_tasks(): + + while True: + for task in TASKS.copy(): + if task.done(): + continue + + # these are the asyncio tasks. Exceptions are handled either in Task or DeviceTask, + # so we can not await the tasks here because that would raise the Exception. + # That's why we use an event to signal that the task is done + event = Event() + task.add_done_callback(lambda x: event.set()) + await event.wait() + break + + else: + break + + log.debug('All tasks done') + + +class Task: + def __init__(self, coro: Callable[[], Awaitable], *, name: str): + self._coro: Final = coro + self._name: Final = name + + self._task: asyncio_Task | None = None + + @property + def is_running(self) -> bool: + if (task := self._task) is None or task.cancelled(): + return False + return True + + def start(self): + if not self.is_running: + self._task = create_task(self._wrapper(), name=self._name) + + def cancel(self) -> asyncio_Task | None: + if (task := self._task) is None: + return None + + task.cancel() + return task + + async def cancel_and_wait(self) -> bool: + if (task := self.cancel()) is None: + return False + + try: # noqa: SIM105 + await task + except CancelledError: + pass + return True + + async def _wrapper(self): + task = current_task() + + try: + await self._coro() + except Exception as e: + self.process_exception(e) + except CancelledError: + pass + finally: + if task is self._task: + self._task = None + + log.debug(f'{self._name:s} finished!') + + def process_exception(self, e: Exception): + log.error(f'Error in {self._name:s}') + for line in traceback.format_exc().splitlines(): + log.error(line) + + +class DeviceTask(Task): + def __init__(self, device: DeviceProto, coro: Callable[[], Coroutine], *, name: str): + super().__init__(coro, name=name) + self._device: Final = device + + def process_exception(self, e: Exception): + super().process_exception(e) + self._device.on_source_failed(f'Task crashed: {e.__class__.__name__}') diff --git a/src/sml2mqtt/const/time_series.py b/src/sml2mqtt/const/time_series.py new file mode 100644 index 0000000..0ff599a --- /dev/null +++ b/src/sml2mqtt/const/time_series.py @@ -0,0 +1,77 @@ +from collections import deque +from collections.abc import Sequence +from datetime import timedelta +from typing import Final, TypeAlias + + +DurationType: TypeAlias = timedelta | float | int + + +def get_duration(obj: DurationType) -> int | float: + if isinstance(obj, timedelta): + return int(obj.total_seconds()) if not obj.microseconds else obj.total_seconds() + if not isinstance(obj, (int, float)): + raise TypeError() + return obj + + +class TimeSeries: + __slots__ = ('period', 'times', 'values', 'is_full', 'wait_for_data') + + def __init__(self, period: DurationType, wait_for_data: bool = False): + self.wait_for_data: Final = wait_for_data + self.period: Final = get_duration(period) + self.times: Final[deque[float]] = deque() + self.values: Final[deque[float]] = deque() + + self.is_full: bool = False + + def clear(self): + self.is_full = False + self.times.clear() + self.values.clear() + + def add_value(self, value: int | float | None, timestamp: float): + start = timestamp - self.period + + if value is not None: + self.times.append(timestamp) + self.values.append(value) + + if not self.is_full and self.times and self.times[0] <= start: + self.is_full = True + + try: + while self.times[1] <= start: + self.times.popleft() + self.values.popleft() + except IndexError: + pass + + def get_values(self) -> Sequence[float] | None: + if not self.values or self.wait_for_data and not self.is_full: + return None + return tuple(self.values) + + def get_value_duration(self, timestamp: float) -> Sequence[tuple[float, float]] | None: + if not self.values or self.wait_for_data and not self.is_full: + return None + + start_of_interval = timestamp - self.period + + start = self.times[0] + if start <= start_of_interval: + start = start_of_interval + + stop = timestamp + value = self.values[0] + + ret = [] + for i in range(1, len(self.values)): + stop = self.times[i] + ret.append((value, stop - start)) + start = stop + value = self.values[i] + + ret.append((value, stop - start)) + return ret diff --git a/src/sml2mqtt/device/__init__.py b/src/sml2mqtt/device/__init__.py deleted file mode 100644 index 7ea090d..0000000 --- a/src/sml2mqtt/device/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -from sml2mqtt.device.sml_device_status import DeviceStatus - -# isort: split - -from sml2mqtt.device.sml_device import Device -from sml2mqtt.device.sml_serial import SmlSerial diff --git a/src/sml2mqtt/device/sml_device.py b/src/sml2mqtt/device/sml_device.py deleted file mode 100644 index 55c57a4..0000000 --- a/src/sml2mqtt/device/sml_device.py +++ /dev/null @@ -1,294 +0,0 @@ -import logging -import traceback -from asyncio import Event -from binascii import b2a_hex -from typing import Dict, Final, List, Optional, Set - -from smllib import SmlFrame, SmlStreamReader -from smllib.errors import CrcError -from smllib.sml import SmlListEntry - -import sml2mqtt -from sml2mqtt import CMD_ARGS -from sml2mqtt.__log__ import get_logger -from sml2mqtt.__shutdown__ import shutdown -from sml2mqtt.config import CONFIG -from sml2mqtt.config.config import PortSettings -from sml2mqtt.config.device import SmlDeviceConfig, SmlValueConfig -from sml2mqtt.device import DeviceStatus -from sml2mqtt.device.watchdog import Watchdog -from sml2mqtt.errors import AllDevicesFailedError, DeviceSetupFailedError, \ - ObisIdForConfigurationMappingNotFoundError, Sml2MqttConfigMappingError -from sml2mqtt.mqtt import MqttObj -from sml2mqtt.sml_value import SmlValue - -Event().set() - -ALL_DEVICES: Dict[str, 'Device'] = {} - - -class Device: - @classmethod - async def create(cls, settings: PortSettings, timeout: float, skip_values: Set[str], mqtt_device: MqttObj): - device = None - try: - device = cls(settings.url, timeout, set(skip_values), mqtt_device) - device.serial = await sml2mqtt.device.SmlSerial.create(settings, device) - ALL_DEVICES[settings.url] = device - - return device - except Exception as e: - if device is None: - get_logger('device').error('Setup failed!') - else: - device.log.error('Setup failed') - raise DeviceSetupFailedError(e) from None - - def __init__(self, url: str, timeout: float, skip_values: Set[str], mqtt_device: MqttObj): - self.stream = SmlStreamReader() - self.serial: 'sml2mqtt.device.SmlSerial' = None - self.watchdog: Final = Watchdog(timeout, self.serial_data_timeout) - - self.status: DeviceStatus = DeviceStatus.STARTUP - self.mqtt_device: MqttObj = mqtt_device - self.mqtt_status: MqttObj = mqtt_device.create_child('status') - - self.log = get_logger(url.split("/")[-1]) - self.log_status = get_logger(url.split("/")[-1]).getChild('status') - - self.device_url = url - self.device_id: str = url.split("/")[-1] - - self.sml_values: Dict[str, SmlValue] = {} - - self.skip_values = skip_values - - def start(self): - self.serial.start() - self.watchdog.start() - - def stop(self): - self.serial.cancel() - self.watchdog.cancel() - - def __await__(self): - yield from self.serial.wait_for_cancel().__await__() - yield from self.watchdog.wait_for_cancel().__await__() - - def shutdown(self): - if not self.status.is_shutdown_status(): - self.set_status(DeviceStatus.SHUTDOWN) - - def set_status(self, new_status: DeviceStatus) -> bool: - if self.status == new_status: - return False - - self.status = new_status - self.log_status.info(f'{new_status:s}') - - # Don't publish the port open because we don't have the correct name from the config yet - if new_status != DeviceStatus.PORT_OPENED: - self.mqtt_status.publish(new_status.value) - - # If all ports are closed, or we have errors we shut down - if all(x.status.is_shutdown_status() for x in ALL_DEVICES.values()): - # Stop reading from the serial port because we are shutting down - self.serial.close() - self.watchdog.cancel() - shutdown(AllDevicesFailedError) - return True - - def _select_device_id(self, frame_values: Dict[str, SmlListEntry]) -> str: - # search frame and see if we get a match - for search_obis in CONFIG.general.device_id_obis: - if (obis_value := frame_values.get(search_obis)) is not None: - self.log.debug(f'Found obis id {search_obis:s} in the sml frame') - value = obis_value.get_value() - self.device_id = str(value) - return str(search_obis) - - searched = ', '.join(CONFIG.general.device_id_obis) - self.log.error(f'Found none of the following obis ids in the sml frame: {searched:s}') - raise ObisIdForConfigurationMappingNotFoundError() - - def _select_device_config(self) -> Optional[SmlDeviceConfig]: - device_cfg = CONFIG.devices.get(self.device_id) - if device_cfg is None: - self.log.warning(f'No configuration found for {self.device_id:s}') - return None - - self.log.debug(f'Configuration found for {self.device_id:s}') - return device_cfg - - def _setup_device(self, frame_values: Dict[str, SmlListEntry]): - found_obis = self._select_device_id(frame_values) - cfg = self._select_device_config() - - # Global configuration option to ignore mapping value - if not CONFIG.general.report_device_id: - self.skip_values.add(found_obis) - - # Change the mqtt topic default from device url to the matched device id - self.mqtt_device.set_topic(self.device_id) - - # override from config - if cfg is not None: - # setup topics - self.mqtt_device.set_config(cfg.mqtt) - self.mqtt_status.set_config(cfg.status) - - # additional obis values that are ignored from the config - if cfg.skip is not None: - self.skip_values.update(cfg.skip) - - self._setup_sml_values(cfg, frame_values) - - def _setup_sml_values(self, device_config: Optional[SmlDeviceConfig], frame_values: Dict[str, SmlListEntry]): - log_level = logging.DEBUG if not CMD_ARGS.analyze else logging.INFO - values_config: Dict[str, SmlValueConfig] = device_config.values if device_config is not None else {} - - from_frame = frozenset(frame_values) - self.skip_values - from_config = frozenset(values_config) - default_cfg = from_frame - from_config - - # create entries where we don't have a user config - for obis_id in sorted(default_cfg): - self.log.log(log_level, f'Creating default value handler for {obis_id}') - self.sml_values[obis_id] = SmlValue( - self.device_id, obis_id, self.mqtt_device.create_child(obis_id), - workarounds=[], transformations=[], filters=sml2mqtt.sml_value.filter_from_config(None) - ) - - # create entries which are defined by the user - for obis_id, value_config in sorted(values_config.items()): - # little helper to catch config errors - if obis_id in self.skip_values: - self.log.warning(f'Config for {obis_id:s} found but {obis_id:s} is also marked to be skipped') - if obis_id not in frame_values: - self.log.warning(f'Config for {obis_id:s} found but {obis_id:s} was not reported by the frame') - - self.log.log(log_level, f'Creating value handler from config for {obis_id}') - self.sml_values[obis_id] = SmlValue( - self.device_id, obis_id, self.mqtt_device.create_child(obis_id).set_config(value_config.mqtt), - workarounds=sml2mqtt.sml_value.workaround_from_config(value_config.workarounds), - transformations=sml2mqtt.sml_value.transform_from_config(value_config.transformations), - filters=sml2mqtt.sml_value.filter_from_config(value_config.filters), - ) - - def serial_data_timeout(self): - if self.set_status(DeviceStatus.MSG_TIMEOUT): - self.stream.clear() - self.log.warning('Timeout') - - def serial_data_read(self, data: bytes): - frame = None - - try: - self.watchdog.feed() - self.stream.add(data) - - try: - frame = self.stream.get_frame() - if frame is None: - return None - except CrcError as e: - self.log.error(f'Crc error: {e.crc_calc} != {e.crc_msg}') - self.set_status(DeviceStatus.CRC_ERROR) - return None - - # Process Frame - self.process_frame(frame) - except Exception as e: - # dump frame if possible - if frame is not None: - self.log.error('Received Frame') - self.log.error(f' -> {b2a_hex(frame.buffer)}') - - # Log exception - if isinstance(e, Sml2MqttConfigMappingError): - self.log.error(str(e)) - else: - for line in traceback.format_exc().splitlines(): - self.log.error(line) - - # Signal that an error occurred - self.set_status(DeviceStatus.ERROR) - return None - - def process_frame(self, frame: SmlFrame): - - do_analyze = sml2mqtt.CMD_ARGS.analyze - do_wh_in_kwh = CONFIG.general.wh_in_kwh - report_blank_meters = CONFIG.general.report_blank_energy_meters - - if do_analyze: - self.log.info('') - self.log.info('Received Frame') - self.log.info(f' -> {b2a_hex(frame.buffer)}') - self.log.info('') - for obj in frame.parse_frame(): - for line in obj.format_msg().splitlines(): - self.log.info(line) - self.log.info('') - - # try shortcut, if that fails try parsing the whole frame - try: - sml_objs: List[SmlListEntry] = frame.get_obis() - except Exception: - self.log.info('get_obis failed - try parsing frame') - for line in traceback.format_exc().splitlines(): - self.log.debug(line) - - sml_objs: List[SmlListEntry] = [] - for msg in frame.parse_frame(): - for val in getattr(msg.message_body, 'val_list', []): - sml_objs.append(val) - - frame_values: Dict[str, SmlListEntry] = {} - for sml_obj in sml_objs: - name = sml_obj.obis - if name in self.skip_values: - continue - - # Unit is Wh -> Value is from an Energy Meter - if sml_obj.unit == 30: - # We Don't publish disabled energy meters (always 0) - if sml_obj.get_value() < 0.1 and not report_blank_meters: - continue - # Global option to automatically convert from Wh to kWh - if do_wh_in_kwh: - sml_obj.value /= 1000 - - # Mark for publishing - frame_values[name] = sml_obj - - # If we don't have the values we have to set up the device first - if not self.sml_values: - self._setup_device(frame_values) - for drop_obis in self.skip_values: - frame_values.pop(drop_obis, None) - - for obis_id, frame_value in frame_values.items(): - if (sml_value := self.sml_values.get(obis_id)) is not None: - sml_value.set_value(frame_value, frame_values) - else: - # This can only happen if the device does not report all values with the initial frame - # The user then has to skip the obis ids or manually add them to the configuration - self.log.error('Unexpected obis id received!') - for line in frame_value.format_msg().splitlines(): - self.log.error(line) - self.set_status(DeviceStatus.ERROR) - return None - - # There was no Error -> OK - self.set_status(DeviceStatus.OK) - - if do_analyze: - for value in self.sml_values.values(): - self.log.info('') - for line in value.describe().splitlines(): - self.log.info(line) - - self.log.info('') - self.set_status(DeviceStatus.SHUTDOWN) - return None diff --git a/src/sml2mqtt/device/sml_serial.py b/src/sml2mqtt/device/sml_serial.py deleted file mode 100644 index 4181fc3..0000000 --- a/src/sml2mqtt/device/sml_serial.py +++ /dev/null @@ -1,103 +0,0 @@ -import asyncio -from asyncio import CancelledError, create_task, Task -from time import monotonic -from typing import Optional, TYPE_CHECKING - -from serial_asyncio import create_serial_connection, SerialTransport - -from sml2mqtt.__log__ import get_logger -from sml2mqtt.config.config import PortSettings -from sml2mqtt.device import DeviceStatus - -if TYPE_CHECKING: - import sml2mqtt - - -log = get_logger('serial') - - -class SmlSerial(asyncio.Protocol): - @classmethod - async def create(cls, settings: PortSettings, device: 'sml2mqtt.device_id.Device') -> 'SmlSerial': - transport, protocol = await create_serial_connection( - asyncio.get_event_loop(), cls, - url=settings.url, - baudrate=settings.baudrate, parity=settings.parity, - stopbits=settings.stopbits, bytesize=settings.bytesize) # type: SerialTransport, SmlSerial - - protocol.url = settings.url - protocol.device = device - return protocol - - def __init__(self) -> None: - super().__init__() - - self.url: Optional[str] = None - self.device: Optional['sml2mqtt.device_id.Device'] = None - - self.transport: Optional[SerialTransport] = None - - self.task: Optional[Task] = None - self.last_read: Optional[float] = None - - def connection_made(self, transport: SerialTransport): - self.transport = transport - log.debug(f'Port {self.url} successfully opened') - - self.device.set_status(DeviceStatus.PORT_OPENED) - - # so we can read bigger chunks at once in case someone uses a higher baudrate - self.transport._max_read_size = 10_240 - - def connection_lost(self, exc): - self.close() - - log.info(f'Port {self.url} was closed') - self.device.set_status(DeviceStatus.PORT_CLOSED) - - def data_received(self, data: bytes): - self.transport.pause_reading() - self.last_read = monotonic() - - self.device.serial_data_read(data) - - async def _chunk_task(self): - interval = 0.2 - - while True: - await asyncio.sleep(interval) - - if self.last_read is not None: - diff_to_interval = interval - (monotonic() - self.last_read) - self.last_read = None - if diff_to_interval >= 0.001: - await asyncio.sleep(diff_to_interval) - - self.transport.resume_reading() - - def start(self): - assert self.task is None - self.task = create_task(self._chunk_task(), name=f'Chunk task {self.url:s}') - - def cancel(self): - self.close() - - async def wait_for_cancel(self): - if self.task is None: - return False - try: - await self.task - except CancelledError: - pass - return True - - def close(self): - if not self.transport.is_closing(): - self.transport.close() - - if (task := self.task) is None: - return None - - task.cancel() - self.task = None - return task diff --git a/src/sml2mqtt/device/sml_value_group.py b/src/sml2mqtt/device/sml_value_group.py deleted file mode 100644 index 8ede679..0000000 --- a/src/sml2mqtt/device/sml_value_group.py +++ /dev/null @@ -1,19 +0,0 @@ -from typing import Dict, Final - -from smllib.sml import SmlListEntry - -from sml2mqtt.sml_value import SmlValue - - -class SmlValueGroup: - def __init__(self, name: str): - self.name: Final = name - self.values: Dict[str, SmlValue] = {} - - def process_frame(self, frame_values: Dict[str, SmlListEntry]): - for obis, frame_value in frame_values.items(): - value = self.values[obis] - value.set_value(frame_value, frame_values) - - def __str__(self) -> str: - return f'<{self.__class__.__name__} {self.name:s}>' diff --git a/src/sml2mqtt/device/watchdog.py b/src/sml2mqtt/device/watchdog.py deleted file mode 100644 index 98cd4d5..0000000 --- a/src/sml2mqtt/device/watchdog.py +++ /dev/null @@ -1,53 +0,0 @@ -from asyncio import CancelledError, create_task, Event, Task, TimeoutError, wait_for -from typing import Any, Callable, Final, Optional - - -class Watchdog: - def __init__(self, timeout: float, callback: Callable[[], Any]): - if timeout <= 0: - raise ValueError() - self.timeout: Final = timeout - self.callback: Final = callback - self.event: Final = Event() - self.task: Optional[Task] = None - - def start(self): - assert self.task is None - self.task = create_task(self.wd_task()) - - def cancel(self): - if self.task is not None: - self.task.cancel() - self.task = None - - async def wait_for_cancel(self): - if self.task is None: - return False - try: - await self.task - except CancelledError: - pass - return True - - def feed(self): - self.event.set() - - async def wd_task(self): - try: - make_call = True - while True: - self.event.clear() - - try: - await wait_for(self.event.wait(), self.timeout) - make_call = True - continue - except TimeoutError: - pass - - # callback only once! - if make_call: - make_call = False - self.callback() - finally: - self.task = None diff --git a/src/sml2mqtt/errors.py b/src/sml2mqtt/errors.py index 0b1241c..71ab5dc 100644 --- a/src/sml2mqtt/errors.py +++ b/src/sml2mqtt/errors.py @@ -1,8 +1,26 @@ -class Sml2MqttException(Exception): # noqa: N818 +from __future__ import annotations + +from typing import TYPE_CHECKING, Final + +from typing_extensions import override + + +if TYPE_CHECKING: + from logging import Logger + + from smllib.sml import SmlListEntry + + +class Sml2MqttException(Exception): pass -class AllDevicesFailedError(Sml2MqttException): +class Sml2MqttExceptionWithLog(Sml2MqttException): + def log_msg(self, log: Logger): + raise NotImplementedError() + + +class DeviceFailedError(Sml2MqttException): pass @@ -28,5 +46,55 @@ class Sml2MqttConfigMappingError(Sml2MqttException): pass -class ObisIdForConfigurationMappingNotFoundError(Sml2MqttConfigMappingError): - pass +class ObisIdForConfigurationMappingNotFoundError(Sml2MqttExceptionWithLog): + @override + def log_msg(self, log: Logger): + return None + + +# ------------------------------------------------------------------------------------ +# Source Errors +# ------------------------------------------------------------------------------------ + +class HttpStatusError(Sml2MqttExceptionWithLog): + def __init__(self, status: int): + super().__init__() + self.status: Final = status + + def __str__(self): + return f'{self.__class__.__name__:s}: {self.status:d}' + + @override + def log_msg(self, log: Logger): + log.error(f'Received http status {self.status}') + + def __eq__(self, other): + if isinstance(other, HttpStatusError): + return self.status == other.status + return NotImplemented + + +# ------------------------------------------------------------------------------------ +# Value Processing Errors +# ------------------------------------------------------------------------------------ +class UnprocessedObisValuesReceivedError(Sml2MqttExceptionWithLog): + def __init__(self, *values: SmlListEntry): + super().__init__() + self.values: Final = values + + @override + def log_msg(self, log: Logger): + log.error(f'Unexpected obis id{"" if len(self.values) == 1 else "s"} received!') + for value in self.values: + for line in value.format_msg().splitlines(): + log.error(line) + + +class RequiredObisValueNotInFrameError(Sml2MqttExceptionWithLog): + def __init__(self, *obis: str): + super().__init__() + self.obis: Final = obis + + @override + def log_msg(self, log: Logger): + log.error(f'Expected obis id{"" if len(self.obis) == 1 else "s"} missing in frame: {", ".join(self.obis)}!') diff --git a/src/sml2mqtt/mqtt/__init__.py b/src/sml2mqtt/mqtt/__init__.py index b84870a..59e7f72 100644 --- a/src/sml2mqtt/mqtt/__init__.py +++ b/src/sml2mqtt/mqtt/__init__.py @@ -1,6 +1,7 @@ from .connect_delay import DynDelay -from .mqtt import cancel, publish, start, wait_for_connect, wait_for_disconnect +from .mqtt import publish, start, wait_for_connect + # isort: split -from .mqtt_obj import BASE_TOPIC, MqttObj, patch_analyze, setup_base_topic +from .mqtt_obj import BASE_TOPIC, MqttObj, check_for_duplicate_topics, patch_analyze, setup_base_topic diff --git a/src/sml2mqtt/mqtt/connect_delay.py b/src/sml2mqtt/mqtt/connect_delay.py index b2cb9c8..272bd3b 100644 --- a/src/sml2mqtt/mqtt/connect_delay.py +++ b/src/sml2mqtt/mqtt/connect_delay.py @@ -1,13 +1,14 @@ from asyncio import sleep -from typing import Optional class DynDelay: - def __init__(self, min_delay: float, max_delay: float, start_delay: Optional[float] = None): + def __init__(self, min_delay: float, max_delay: float, start_delay: float | None = None): if min_delay < 0: - raise ValueError(f'min_delay must be >= 0: {min_delay}') + msg = f'min_delay must be >= 0: {min_delay}' + raise ValueError(msg) if max_delay <= min_delay: - raise ValueError(f'max_delay must be >= min_delay: {max_delay}') + msg = f'max_delay must be >= min_delay: {max_delay}' + raise ValueError(msg) self.min = min_delay self.max = max_delay diff --git a/src/sml2mqtt/mqtt/errors.py b/src/sml2mqtt/mqtt/errors.py index 8ad7fda..792ff76 100644 --- a/src/sml2mqtt/mqtt/errors.py +++ b/src/sml2mqtt/mqtt/errors.py @@ -6,5 +6,9 @@ class TopicFragmentExpectedError(Exception): pass +class MqttTopicEmpty(Exception): + pass + + class MqttConfigValuesMissingError(Exception): pass diff --git a/src/sml2mqtt/mqtt/mqtt.py b/src/sml2mqtt/mqtt/mqtt.py index 6a4983f..0d161ab 100644 --- a/src/sml2mqtt/mqtt/mqtt.py +++ b/src/sml2mqtt/mqtt/mqtt.py @@ -1,40 +1,39 @@ import traceback -from asyncio import CancelledError, create_task, Event, Queue, Task, TimeoutError, wait_for -from typing import Final, Optional, Union +from asyncio import CancelledError, Event, Queue, TimeoutError, wait_for +from typing import Final -from asyncio_mqtt import Client, MqttError, Will +from aiomqtt import Client, MqttError, Will import sml2mqtt from sml2mqtt.__log__ import log as _parent_logger +from sml2mqtt.const import Task from sml2mqtt.errors import InitialMqttConnectionFailedError from sml2mqtt.mqtt import DynDelay +from sml2mqtt.runtime import on_shutdown -log = _parent_logger.getChild('mqtt') +log = _parent_logger.getChild('mqtt') -TASK: Optional[Task] = None -IS_CONNECTED: Optional[Event] = None +TASK: Task | None = None +IS_CONNECTED: Event | None = None -def start(): - global TASK, IS_CONNECTED - IS_CONNECTED = Event() +async def start(): + global IS_CONNECTED, TASK assert TASK is None - TASK = create_task(mqtt_task(), name='MQTT Task') + IS_CONNECTED = Event() + TASK = Task(_mqtt_task, name='MQTT Task') -def cancel(): - global TASK - if TASK is not None: - TASK.cancel() - TASK = None + on_shutdown(TASK.cancel_and_wait, 'Shutdown mqtt') + TASK.start() async def wait_for_connect(timeout: float): if IS_CONNECTED is None: - return None + raise ValueError() try: await wait_for(IS_CONNECTED.wait(), timeout) @@ -45,24 +44,10 @@ async def wait_for_connect(timeout: float): return None -async def wait_for_disconnect(): - if TASK is None: - return None - - await TASK - - -QUEUE: Optional[Queue] = None - - -async def mqtt_task(): - try: - await _mqtt_task() - finally: - log.debug('Task finished') +QUEUE: Queue[tuple[str, int | float | str | bytes, int, bool]] | None = None -async def _mqtt_task(): +async def _mqtt_task() -> None: global QUEUE from .mqtt_obj import BASE_TOPIC @@ -72,8 +57,8 @@ async def _mqtt_task(): delay = DynDelay(0, 300) - payload_offline: Final = 'OFFLINE' payload_online: Final = 'ONLINE' + payload_offline: Final = 'OFFLINE' shutdown = False @@ -82,16 +67,24 @@ async def _mqtt_task(): try: # since we just pass this into the mqtt wrapper we do not link it to the base topic - will_topic = BASE_TOPIC.create_child( - topic_fragment=config.mqtt.last_will.topic).set_config(config.mqtt.last_will) + will_topic = BASE_TOPIC.create_child(topic_fragment='status').set_config(config.mqtt.last_will) + + will = Will( + topic=will_topic.topic, payload=payload_offline, + qos=will_topic.qos, retain=will_topic.retain + ) + + tls_kwargs = {} if cfg_connection.tls is None else cfg_connection.tls.get_client_kwargs(log) client = Client( - hostname=cfg_connection.host, - port=cfg_connection.port, + hostname=cfg_connection.host, port=cfg_connection.port, + username=cfg_connection.user if cfg_connection.user else None, password=cfg_connection.password if cfg_connection.password else None, - will=Will(will_topic.topic, payload=payload_offline, qos=will_topic.qos, retain=will_topic.retain), - client_id=cfg_connection.client_id + will=will, + identifier=cfg_connection.identifier, + + **tls_kwargs ) log.debug(f'Connecting to {cfg_connection.host}:{cfg_connection.port}') @@ -104,31 +97,35 @@ async def _mqtt_task(): try: # signal that we are online - await client.publish(will_topic.topic, payload_online, will_topic.qos, will_topic.retain) + await client.publish(will.topic, payload_online, will.qos, will.retain) # worker to publish things while True: topic, value, qos, retain = await QUEUE.get() await client.publish(topic, value, qos, retain) QUEUE.task_done() + except CancelledError: # The last will testament only gets sent on abnormal disconnect # Since we disconnect gracefully we have to manually sent the offline status - await client.publish(will_topic.topic, payload_offline, will_topic.qos, will_topic.retain) + await client.publish(will.topic, will.payload, will.qos, will.retain) shutdown = True + log.debug('Disconnecting') except MqttError as e: delay.increase() log.error(f'{e} ({e.__class__.__name__})') + except Exception: delay.increase() for line in traceback.format_exc().splitlines(): log.error(line) + finally: QUEUE = None IS_CONNECTED.clear() -def publish(topic: str, value: Union[int, float, str], qos: int, retain: bool): +def publish(topic: str, value: int | float | str | bytes, qos: int, retain: bool): if QUEUE is not None: QUEUE.put_nowait((topic, value, qos, retain)) diff --git a/src/sml2mqtt/mqtt/mqtt_obj.py b/src/sml2mqtt/mqtt/mqtt_obj.py index 2932060..cd0a08e 100644 --- a/src/sml2mqtt/mqtt/mqtt_obj.py +++ b/src/sml2mqtt/mqtt/mqtt_obj.py @@ -1,27 +1,35 @@ import dataclasses -from typing import Any, Callable, Final, List, Optional, Union +from collections.abc import Callable, Generator +from typing import Any, Final from sml2mqtt.__log__ import get_logger from sml2mqtt.config import OptionalMqttPublishConfig from sml2mqtt.mqtt import publish -from .errors import MqttConfigValuesMissingError, TopicFragmentExpectedError +from .errors import MqttConfigValuesMissingError, MqttTopicEmpty, TopicFragmentExpectedError -pub_func: Callable[[str, Union[int, float, str], int, bool], Any] = publish +pub_func: Callable[[str, int | float | str, int, bool], Any] = publish -def publish_analyze(topic: str, value: Union[int, float, str], qos: int, retain: bool): + +def publish_analyze(topic: str, value: int | float | str, qos: int, retain: bool): get_logger('mqtt.pub').info(f'{topic}: {value} (QOS: {qos}, retain: {retain})') +def patch_analyze(): + global pub_func + + pub_func = publish_analyze + + @dataclasses.dataclass class MqttCfg: - topic_full: Optional[str] = None - topic_fragment: Optional[str] = None - qos: Optional[int] = None - retain: Optional[bool] = None + topic_full: str | None = None + topic_fragment: str | None = None + qos: int | None = None + retain: bool | None = None - def set_config(self, config: Optional[OptionalMqttPublishConfig]): + def set_config(self, config: OptionalMqttPublishConfig): self.topic_full = config.full_topic self.topic_fragment = config.topic self.qos = config.qos @@ -29,7 +37,7 @@ def set_config(self, config: Optional[OptionalMqttPublishConfig]): class MqttObj: - def __init__(self, topic_fragment: Optional[str] = None, qos: Optional[int] = None, retain: Optional[bool] = None): + def __init__(self, topic_fragment: str | None = None, qos: int | None = None, retain: bool | None = None): # Configured parts self.cfg = MqttCfg(topic_fragment=topic_fragment, qos=qos, retain=retain) @@ -39,10 +47,10 @@ def __init__(self, topic_fragment: Optional[str] = None, qos: Optional[int] = No self.retain: bool = False self.topic: str = '' - self.parent: Optional[MqttObj] = None - self.children: List[MqttObj] = [] + self.parent: MqttObj | None = None + self.children: list[MqttObj] = [] - def publish(self, value: Union[str, int, float, bytes]): + def publish(self, value: str | int | float): pub_func(self.topic, value, self.qos, self.retain) def update(self) -> 'MqttObj': @@ -69,7 +77,17 @@ def _merge_values(self) -> 'MqttObj': else: if not self.cfg.topic_fragment: raise TopicFragmentExpectedError() - self.topic = f'{self.parent.topic}/{self.cfg.topic_fragment}' + + # The topmost topic may be empty + parts = [] + if self.parent.topic: + parts.append(self.parent.topic) + if self.cfg.topic_fragment: + parts.append(self.cfg.topic_fragment) + self.topic = '/'.join(parts) + + if not self.topic: + raise MqttTopicEmpty() # effective QOS self.qos = self.cfg.qos @@ -82,12 +100,12 @@ def _merge_values(self) -> 'MqttObj': self.retain = self.parent.retain return self - def set_topic(self, topic: str) -> 'MqttObj': + def set_topic(self, topic: str | None) -> 'MqttObj': self.cfg.topic_fragment = topic self.update() return self - def set_config(self, cfg: Optional[OptionalMqttPublishConfig]) -> 'MqttObj': + def set_config(self, cfg: OptionalMqttPublishConfig | None) -> 'MqttObj': if cfg is None: return self @@ -95,14 +113,19 @@ def set_config(self, cfg: Optional[OptionalMqttPublishConfig]) -> 'MqttObj': self.update() return self - def create_child(self, topic_fragment: Optional[str] = None, qos: Optional[int] = None, - retain: Optional[bool] = None) -> 'MqttObj': + def create_child(self, topic_fragment: str | None = None, qos: int | None = None, + retain: bool | None = None) -> 'MqttObj': child = self.__class__(topic_fragment=topic_fragment, qos=qos, retain=retain) child.parent = self self.children.append(child) child.update() return child + def iter_objs(self) -> Generator['MqttObj', Any, None]: + yield self + for child in self.children: + yield from child.iter_objs() + BASE_TOPIC: Final = MqttObj() @@ -114,7 +137,11 @@ def setup_base_topic(topic: str, qos: int, retain: bool): BASE_TOPIC.update() -def patch_analyze(): - global pub_func +def check_for_duplicate_topics(obj: MqttObj): + log = get_logger('mqtt') - pub_func = publish_analyze + topics: set[str] = set() + for o in obj.iter_objs(): + if (topic := o.topic) in topics: + log.warning(f'Topic "{topic:s}" is already configured!') + topics.add(topic) diff --git a/src/sml2mqtt/runtime/__init__.py b/src/sml2mqtt/runtime/__init__.py new file mode 100644 index 0000000..18a45f7 --- /dev/null +++ b/src/sml2mqtt/runtime/__init__.py @@ -0,0 +1 @@ +from .shutdown import do_shutdown, do_shutdown_async, on_shutdown, signal_handler_setup diff --git a/src/sml2mqtt/runtime/shutdown.py b/src/sml2mqtt/runtime/shutdown.py new file mode 100644 index 0000000..4f330a6 --- /dev/null +++ b/src/sml2mqtt/runtime/shutdown.py @@ -0,0 +1,87 @@ +from __future__ import annotations + +import logging.handlers +import signal +import traceback +from dataclasses import dataclass +from threading import Lock +from typing import TYPE_CHECKING, Final + +from sml2mqtt.const import Task, create_task + + +if TYPE_CHECKING: + from collections.abc import Awaitable, Callable + + +log = logging.getLogger('sml.shutdown') + + +@dataclass(frozen=True) +class ShutdownObj: + coro: Callable[[], Awaitable] + msg: str + + async def do(self): + try: + log.debug(self.msg) + await self.coro() + log.debug(f'{self.msg:s} done!') + except Exception as e: + log.error(str(e)) + tb = traceback.format_exc().splitlines() + for line in tb: + log.error(line) + + +async def shutdown_coro(): + log.debug('Starting shutdown') + for obj in SHUTDOWN_OBJS: + await obj.do() + log.debug('Shutdown complete') + + +def on_shutdown(coro: Callable[[], Awaitable], msg: str): + global SHUTDOWN_OBJS + for obj in SHUTDOWN_OBJS: + if obj.coro == coro or obj.msg == msg: + raise ValueError() + SHUTDOWN_OBJS = (*SHUTDOWN_OBJS, ShutdownObj(coro, msg)) + + +SHUTDOWN_OBJS: tuple[ShutdownObj, ...] = () +SHUTDOWN_TASK: Final = Task(shutdown_coro, name='Shutdown Task') + +SHUTDOWN_LOCK: Final = Lock() +SHUTDOWN_CALL: Task | None = None + + +async def do_shutdown_async(): + global SHUTDOWN_CALL + + try: + if not SHUTDOWN_TASK.is_running: + print('Shutting down ...') + log.info('Shutting down ...') + SHUTDOWN_TASK.start() + finally: + with SHUTDOWN_LOCK: + SHUTDOWN_CALL = None + + +def do_shutdown(): + global SHUTDOWN_CALL + + with SHUTDOWN_LOCK: + if SHUTDOWN_CALL is not None: + return None + SHUTDOWN_CALL = create_task(do_shutdown_async()) + + +def _signal_handler_shutdown(sig, frame): + do_shutdown() + + +def signal_handler_setup(): + signal.signal(signal.SIGINT, _signal_handler_shutdown) + signal.signal(signal.SIGTERM, _signal_handler_shutdown) diff --git a/src/sml2mqtt/sml_device/__init__.py b/src/sml2mqtt/sml_device/__init__.py new file mode 100644 index 0000000..e468a39 --- /dev/null +++ b/src/sml2mqtt/sml_device/__init__.py @@ -0,0 +1,2 @@ +from .sml_device import DeviceStatus, SmlDevice +from .sml_devices import ALL_DEVICES diff --git a/src/sml2mqtt/device/sml_device_status.py b/src/sml2mqtt/sml_device/device_status.py similarity index 57% rename from src/sml2mqtt/device/sml_device_status.py rename to src/sml2mqtt/sml_device/device_status.py index ff10c36..39345ff 100644 --- a/src/sml2mqtt/device/sml_device_status.py +++ b/src/sml2mqtt/sml_device/device_status.py @@ -4,15 +4,16 @@ class DeviceStatus(str, Enum): STARTUP = 'STARTUP' SHUTDOWN = 'SHUTDOWN' - PORT_OPENED = 'PORT_OPENED' - PORT_CLOSED = 'PORT_CLOSED' - MSG_TIMEOUT = 'MSG_TIMEOUT' + + SOURCE_FAILED = 'SOURCE_FAILED' + MSG_TIMEOUT = 'MESSAGE_TIMEOUT' CRC_ERROR = 'CRC_ERROR' ERROR = 'ERROR' + OK = 'OK' def is_shutdown_status(self) -> bool: - return self.value in (DeviceStatus.PORT_CLOSED, DeviceStatus.ERROR, DeviceStatus.SHUTDOWN) + return self.value in (DeviceStatus.ERROR, DeviceStatus.SOURCE_FAILED, DeviceStatus.SHUTDOWN) def __str__(self) -> str: return self.value diff --git a/src/sml2mqtt/sml_device/setup_device.py b/src/sml2mqtt/sml_device/setup_device.py new file mode 100644 index 0000000..ade8f66 --- /dev/null +++ b/src/sml2mqtt/sml_device/setup_device.py @@ -0,0 +1,123 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from sml2mqtt.mqtt import BASE_TOPIC, check_for_duplicate_topics +from sml2mqtt.sml_value import SmlValue +from sml2mqtt.sml_value.operations import ( + FactorOperation, + OffsetOperation, + OnChangeFilterOperation, + OrOperation, + RangeFilterOperation, + RefreshActionOperation, + RoundOperation, + SequenceOperation, + SkipZeroMeterOperation, +) +from sml2mqtt.sml_value.setup_operations import setup_operations + + +if TYPE_CHECKING: + import logging + + from sml2mqtt.config.config import GeneralSettings + from sml2mqtt.config.device import SmlDeviceConfig + from sml2mqtt.const import SmlFrameValues + from sml2mqtt.sml_device import SmlDevice + from sml2mqtt.sml_value.base import OperationContainerBase, ValueOperationBase + + +def has_operation_type(obj: OperationContainerBase, *ops: type[ValueOperationBase], + is_of: bool = True) -> ValueOperationBase | None: + for op in obj.operations: + if isinstance(op, ops) == is_of: + return op + if (isinstance(op, (OrOperation, SequenceOperation)) and + (ret := has_operation_type(op, is_of=is_of)) is not None): + return ret + return None + + +def _create_default_transformations(log: logging.Logger, sml_value: SmlValue, frame: SmlFrameValues, + general_cfg: GeneralSettings): + + op_count = len(sml_value.operations) + + if (entry := frame.get_value(sml_value.obis)) is not None and entry.unit == 30: + if general_cfg.wh_in_kwh: + if op := has_operation_type(sml_value, FactorOperation): + log.debug(f'Found {op.__class__.__name__:s} - skip creating default factor') + else: + sml_value.insert_operation(FactorOperation(1 / 1000)) + + # If the user has created something for the meter we don't skip it + if not op_count: + sml_value.insert_operation(SkipZeroMeterOperation()) + + +def _create_default_filters(log: logging.Logger, sml_value: SmlValue, general_cfg: GeneralSettings): + if op := has_operation_type( + sml_value, + FactorOperation, OffsetOperation, RoundOperation, RangeFilterOperation, SkipZeroMeterOperation, + is_of=False + ): + log.debug(f'Found {op.__class__.__name__:s} - skip creating default filters') + return None + + log.info(f'No filters found for {sml_value.obis}, creating default filters') + + sml_value.add_operation(OnChangeFilterOperation()) + sml_value.add_operation(RefreshActionOperation(general_cfg.republish_after)) + + +def setup_device(device: SmlDevice, frame: SmlFrameValues, cfg: SmlDeviceConfig | None, general_cfg: GeneralSettings): + mqtt_device = device.mqtt_device + skip_default_setup = set() + + mqtt_device.set_topic(device.device_id) + + if cfg is not None: + # mqtt of device + mqtt_device.set_config(cfg.mqtt) + device.mqtt_status.set_config(cfg.status) + + skipped_obis = set(cfg.skip) + if not general_cfg.report_device_id and device.device_id is not None: + skipped_obis.update(general_cfg.device_id_obis) + + device.sml_values.set_skipped(*skipped_obis) + skip_default_setup.update(skipped_obis) + + for value_cfg in cfg.values: + obis = value_cfg.obis + skip_default_setup.add(obis) + + if obis in cfg.skip: + device.log.warning(f'Config for {obis:s} found but {obis:s} is also marked to be skipped') + if obis not in frame.obis_ids(): + device.log.warning(f'Config for {obis:s} found but {obis:s} was not reported by the frame') + + sml_value = SmlValue( + obis, + mqtt_device.create_child(topic_fragment=obis).set_config(value_cfg.mqtt) + ) + device.sml_values.add_value(sml_value) + + setup_operations(sml_value, value_cfg) + _create_default_transformations(device.log, sml_value, frame, general_cfg) + _create_default_filters(device.log, sml_value, general_cfg) + else: + # No config found -> ignore defaults + device.sml_values.set_skipped(*general_cfg.device_id_obis) + + # Create default for not + for obis, _ in frame.items(skip=skip_default_setup): # noqa: PERF102 + sml_value = SmlValue(obis, mqtt_device.create_child(topic_fragment=obis)) + device.sml_values.add_value(sml_value) + + _create_default_transformations(device.log, sml_value, frame, general_cfg) + _create_default_filters(device.log, sml_value, general_cfg) + + # Check for duplicate MQTT topics + check_for_duplicate_topics(BASE_TOPIC) diff --git a/src/sml2mqtt/sml_device/sml_device.py b/src/sml2mqtt/sml_device/sml_device.py new file mode 100644 index 0000000..e19bcf6 --- /dev/null +++ b/src/sml2mqtt/sml_device/sml_device.py @@ -0,0 +1,209 @@ +from __future__ import annotations + +import traceback +from logging import DEBUG as LVL_DEBUG +from logging import INFO as LVL_INFO +from typing import TYPE_CHECKING, Any, Final + +import smllib +from smllib import SmlStreamReader +from smllib.errors import CrcError + +from sml2mqtt.__log__ import get_logger +from sml2mqtt.const import EnhancedSmlFrame +from sml2mqtt.errors import ObisIdForConfigurationMappingNotFoundError, Sml2MqttExceptionWithLog +from sml2mqtt.mqtt import BASE_TOPIC +from sml2mqtt.sml_device.sml_devices import ALL_DEVICES +from sml2mqtt.sml_value import SmlValues + +from .. import CONFIG +from .device_status import DeviceStatus +from .setup_device import setup_device +from .watchdog import Watchdog + + +if TYPE_CHECKING: + from collections.abc import Callable + + from sml2mqtt.const import SourceProto + +# ------------------------------------------------------------------------------------------------- +# Dirty: +# Replace the class SmlFrame which is returned by SmlStreamReader with EnhancedSmlFrame +assert hasattr(smllib.reader, 'SmlFrame') +setattr(smllib.reader, 'SmlFrame', EnhancedSmlFrame) + + +class SmlDevice: + def __init__(self, name: str): + self._name: Final = name + self._source: SourceProto | None = None + self.status: DeviceStatus = DeviceStatus.STARTUP + + self.watchdog: Final = Watchdog(self) + self.stream_reader: Final = SmlStreamReader() + + self.log = get_logger(self.name) + self.log_status = self.log.getChild('status') + + self.mqtt_device: Final = BASE_TOPIC.create_child(self.name) + self.mqtt_status: Final = self.mqtt_device.create_child('status') + + self.device_id: str | None = None + self.sml_values: Final = SmlValues() + + self.frame_handler: Callable[[EnhancedSmlFrame], Any] = self.process_first_frame + + @property + def name(self) -> str: + return self._name + + def set_source(self, source: SourceProto): + assert self._source is None, self._source + self._source = source + return self + + async def start(self): + if self._source is not None: + self._source.start() + self.watchdog.start() + + async def cancel_and_wait(self): + if self._source is not None: + await self._source.cancel_and_wait() + await self.watchdog.cancel_and_wait() + + def set_status(self, new_status: DeviceStatus) -> bool: + if (old_status := self.status) == new_status or old_status.is_shutdown_status(): + return False + + self.status = new_status + + # Don't log toggeling between CRC_ERROR and OK. Only log if new status is not OK + level = LVL_INFO + if new_status is DeviceStatus.CRC_ERROR: + level = LVL_DEBUG + elif old_status is DeviceStatus.CRC_ERROR: + if new_status is DeviceStatus.OK: + level = LVL_DEBUG + else: + # Log old status if new status is not OK + self.log_status.log(level, f'Old status {old_status:s}') + + self.log_status.log(level, f'{new_status:s}') + self.mqtt_status.publish(new_status.value) + + ALL_DEVICES.check_status() + return True + + def on_source_data(self, data: bytes): + frame = None # type: EnhancedSmlFrame | None + + try: + self.watchdog.feed() + self.stream_reader.add(data) + + try: + if (frame := self.stream_reader.get_frame()) is None: + return None + except CrcError as e: + self.log.debug(f'Crc error: {e.crc_calc} != {e.crc_msg}') + self.set_status(DeviceStatus.CRC_ERROR) + return None + + # Process Frame + self.frame_handler(frame) + except Exception as e: + # dump frame if possible + if frame is not None: + for line in frame.get_frame_str(): + self.log.info(line) + + self.on_error(e) + + def on_error(self, e: Exception, *, show_traceback: bool = True): + self.log.debug(f'Exception {type(e)}: "{e}"') + + # Log exception + if isinstance(e, Sml2MqttExceptionWithLog): + e.log_msg(self.log) + else: + if show_traceback: + for line in traceback.format_exc().splitlines(): + self.log.error(line) + else: + self.log.error(f'{type(e)}: {e}') + + # Signal that an error occurred + self.set_status(DeviceStatus.ERROR) + return None + + def on_source_failed(self, reason: str): + self.log.error(f'Source failed: {reason}') + self.set_status(DeviceStatus.SOURCE_FAILED) + + def on_timeout(self): + self.set_status(DeviceStatus.MSG_TIMEOUT) + + def process_frame(self, frame: EnhancedSmlFrame): + + frame_values = frame.get_frame_values(self.log) + + self.sml_values.process_frame(frame_values) + + # There was no Error -> OK + self.set_status(DeviceStatus.OK) + + def setup_values_from_frame(self, frame: EnhancedSmlFrame): + frame_values = frame.get_frame_values(self.log) + + # search frame and see if we get a match + for search_obis in CONFIG.general.device_id_obis: + if (obis_value := frame_values.get_value(search_obis)) is not None: + self.log.debug(f'Found obis id {search_obis:s} in the sml frame') + self.device_id = str(obis_value.get_value()) + break + else: + searched = ', '.join(CONFIG.general.device_id_obis) + self.log.error(f'Found none of the following obis ids in the sml frame: {searched:s}') + raise ObisIdForConfigurationMappingNotFoundError() + + # Search configuration to see if we have a special config for the device + device_cfg = CONFIG.devices.get(self.device_id) + if device_cfg is None: + self.log.warning(f'No device found for {self.device_id:s}') + else: + self.log.debug(f'Device found for {self.device_id:s}') + setup_device(self, frame_values, device_cfg, CONFIG.general) + + self.frame_handler = self.process_frame + + def process_first_frame(self, frame: EnhancedSmlFrame): + try: + self.setup_values_from_frame(frame) + except Exception as e: + self.set_status(DeviceStatus.SHUTDOWN) + self.on_error(e) + return None + self.frame_handler(frame) + + def analyze_frame(self, frame: EnhancedSmlFrame): + + # log Frame and frame description + for line in frame.get_analyze_str(): + self.log.info(line) + + # Setup and process the frame + self.setup_values_from_frame(frame) + + # Log setup values + for line in self.sml_values.describe(): + self.log.info(line) + + # process the frame + self.frame_handler(frame) + + # shutdown + self.log.info('') + self.set_status(DeviceStatus.SHUTDOWN) + return None diff --git a/src/sml2mqtt/sml_device/sml_devices.py b/src/sml2mqtt/sml_device/sml_devices.py new file mode 100644 index 0000000..7b864b2 --- /dev/null +++ b/src/sml2mqtt/sml_device/sml_devices.py @@ -0,0 +1,44 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Final + +from sml2mqtt.runtime import do_shutdown + +from .device_status import DeviceStatus + + +if TYPE_CHECKING: + from . import SmlDevice + + +class SmlDevices: + def __init__(self) -> None: + self._devices: tuple[SmlDevice, ...] = () + + def add_device(self, device: SmlDevice) -> SmlDevice: + for existing in self._devices: + if existing.name == device.name: + msg = f'Device {device.name:s} does already exist!' + raise ValueError(msg) + self._devices = (*self._devices, device) + return device + + async def start(self): + for device in self._devices: + await device.start() + + async def cancel_and_wait(self): + for device in self._devices: + await device.cancel_and_wait() + + def check_status(self): + if any(device.status in (DeviceStatus.SOURCE_FAILED, DeviceStatus.SHUTDOWN) for device in self._devices): + return do_shutdown() + + if all(device.status.is_shutdown_status() for device in self._devices): + return do_shutdown() + + return None + + +ALL_DEVICES: Final = SmlDevices() diff --git a/src/sml2mqtt/sml_device/watchdog.py b/src/sml2mqtt/sml_device/watchdog.py new file mode 100644 index 0000000..62fb067 --- /dev/null +++ b/src/sml2mqtt/sml_device/watchdog.py @@ -0,0 +1,59 @@ +from __future__ import annotations + +from asyncio import Event, TimeoutError, wait_for +from typing import TYPE_CHECKING, Final + +from ..const import DeviceTask + + +if TYPE_CHECKING: + from .sml_device import SmlDevice + + +class Watchdog: + def __init__(self, device: SmlDevice): + self._timeout: float = -1 + self.device: Final = device + + self._event: Final = Event() + self._task: Final = DeviceTask(device, self._wd_task, name=f'Watchdog Task {self.device.name:s}') + + def start(self): + self._task.start() + + def cancel(self): + self._task.cancel() + + async def cancel_and_wait(self): + return await self._task.cancel_and_wait() + + def set_timeout(self, timeout: float): + if timeout < 0.1: + raise ValueError() + self._timeout = timeout + return self + + def feed(self): + self._event.set() + + async def _wd_task(self): + make_call = True + while True: + self._event.clear() + + try: + await wait_for(self._event.wait(), self._timeout) + make_call = True + continue + except TimeoutError: + pass + + # callback only once! + if not make_call: + continue + make_call = False + + try: + self.device.on_timeout() + except Exception as e: + self.device.on_error(e) diff --git a/src/sml2mqtt/sml_source/__init__.py b/src/sml2mqtt/sml_source/__init__.py new file mode 100644 index 0000000..4f6fdf7 --- /dev/null +++ b/src/sml2mqtt/sml_source/__init__.py @@ -0,0 +1 @@ +from .setup_source import create_source diff --git a/src/sml2mqtt/sml_source/http.py b/src/sml2mqtt/sml_source/http.py new file mode 100644 index 0000000..5eb9163 --- /dev/null +++ b/src/sml2mqtt/sml_source/http.py @@ -0,0 +1,114 @@ +from __future__ import annotations + +from asyncio import TimeoutError, sleep +from typing import TYPE_CHECKING, Final + +from aiohttp import BasicAuth, ClientError, ClientSession, ClientTimeout + +from sml2mqtt.__log__ import get_logger +from sml2mqtt.const import DeviceTask +from sml2mqtt.errors import HttpStatusError +from sml2mqtt.runtime import on_shutdown + + +if TYPE_CHECKING: + from sml2mqtt.config.inputs import HttpSourceSettings + from sml2mqtt.const import DeviceProto + + +log = get_logger('http') + +SESSION: ClientSession | None = None + + +async def get_session() -> ClientSession: + global SESSION + + if SESSION is not None: + return SESSION + + SESSION = ClientSession() + on_shutdown(close_session, 'Close http session') + return SESSION + + +async def close_session(): + global SESSION + + if (session := SESSION) is None: + return None + + SESSION = None + await session.close() + + # https://docs.aiohttp.org/en/stable/client_advanced.html#graceful-shutdown + await sleep(0.250) + + +class HttpSource: + + @classmethod + async def create(cls, device: DeviceProto, settings: HttpSourceSettings): + auth = None + if settings.user or settings.password: + auth = BasicAuth(settings.user, settings.password) + + return cls(device, str(settings.url), settings.interval, auth, timeout=settings.get_request_timeout()) + + def __init__(self, device: DeviceProto, + url: str, interval: float, + auth: BasicAuth | None, timeout: ClientTimeout) -> None: + super().__init__() + self.device: Final = device + + self.url: Final = url + self.auth: Final = auth + self.timeout: Final = timeout + + self.interval = interval + self._task: Final = DeviceTask(device, self._http_task, name=f'Http Task {self.device.name:s}') + + def start(self) -> None: + self._task.start() + + async def cancel_and_wait(self) -> bool: + return await self._task.cancel_and_wait() + + async def _http_task(self) -> None: + log.debug(f'Requesting data from {self.url}') + + try: + session = await get_session() + except Exception as e: + self.device.on_source_failed(f'Could not create client session: {e}') + return None + + interval: float = self.interval + com_errors: int = 0 + + while True: + await sleep(interval) + interval = self.interval + + try: + resp = await session.get(self.url, auth=self.auth, timeout=self.timeout) + if resp.status != 200: + raise HttpStatusError(resp.status) # noqa: TRY301 + + payload = await resp.read() + com_errors = 0 + except Exception as e: + if isinstance(e, (ClientError, HttpStatusError, TimeoutError)): + com_errors += 1 + max_ignore: int = 7 + if com_errors <= max_ignore: + # errors: 1, 2, 3, 4, 5, 6, 7, 8, 9, ... + # factor: 0.4, 0.9, 1.9, 3.4, 5.4, 7.9, 10.9, 14.4, 18.4, ... + interval = (((com_errors - 0.5) ** 2) / 4 + 0.3) * self.interval + log.debug(f'Ignored {com_errors:d}/{max_ignore:d} {e} ({type(e)})') + continue + + self.device.on_error(e, show_traceback=False) + continue + + self.device.on_source_data(payload) diff --git a/src/sml2mqtt/sml_source/serial.py b/src/sml2mqtt/sml_source/serial.py new file mode 100644 index 0000000..5b9fb01 --- /dev/null +++ b/src/sml2mqtt/sml_source/serial.py @@ -0,0 +1,91 @@ +from __future__ import annotations + +import asyncio +import logging +from asyncio import Protocol +from time import monotonic +from typing import TYPE_CHECKING, Final + +from serial_asyncio import SerialTransport, create_serial_connection + +from sml2mqtt.__log__ import get_logger +from sml2mqtt.const import DeviceTask + + +if TYPE_CHECKING: + from sml2mqtt.config.inputs import SerialSourceSettings + from sml2mqtt.const import DeviceProto + + +log = get_logger('serial') + + +class SerialSource(Protocol): + @classmethod + async def create(cls, device: DeviceProto, settings: SerialSourceSettings) -> SerialSource: + transport, protocol = await create_serial_connection( + asyncio.get_event_loop(), + lambda: cls(device, settings.url), + url=settings.url, + baudrate=settings.baudrate, parity=settings.parity, + stopbits=settings.stopbits, bytesize=settings.bytesize + ) # type: SerialTransport, SerialSource + + return protocol + + def __init__(self, device: DeviceProto, url: str) -> None: + super().__init__() + + self.url: Final = url + self.device: Final = device + + self.transport: SerialTransport | None = None + + self._task: Final = DeviceTask(device, self._chunk_task, name=f'Serial Task {self.device.name:s}') + + self.last_read: float | None = 0.0 + + def start(self): + self._task.start() + + async def cancel_and_wait(self): + return await self._task.cancel_and_wait() + + def connection_made(self, transport: SerialTransport): + self.transport = transport + log.debug(f'Port {self.url:s} successfully opened') + + # so we can read bigger chunks at once in case someone uses a higher baudrate + self.transport._max_read_size = 10_240 + + def connection_lost(self, exc: Exception | None) -> None: + + lvl = logging.INFO + ex_str = '' + if exc is not None: + ex_str = f': {exc}' + lvl = logging.ERROR + + log.log(lvl, f'Port {self.url:s} was closed{ex_str:s}') + self.device.on_source_failed(f'Connection to port {self.url:s} lost') + + def data_received(self, data: bytes): + self.transport.pause_reading() + + self.last_read = monotonic() + self.device.on_source_data(data) + + async def _chunk_task(self): + interval = 0.2 + + while True: + await asyncio.sleep(interval) + + if self.last_read is not None: + diff_to_interval = interval - (monotonic() - self.last_read) + self.last_read = None + if diff_to_interval >= 0.001: + await asyncio.sleep(diff_to_interval) + + # safe to be called multiple times in a row + self.transport.resume_reading() diff --git a/src/sml2mqtt/sml_source/setup_source.py b/src/sml2mqtt/sml_source/setup_source.py new file mode 100644 index 0000000..fdbea11 --- /dev/null +++ b/src/sml2mqtt/sml_source/setup_source.py @@ -0,0 +1,16 @@ + +from sml2mqtt.config.inputs import HttpSourceSettings, SerialSourceSettings +from sml2mqtt.const import DeviceProto, SourceProto + + +async def create_source(device: DeviceProto, settings: SerialSourceSettings | HttpSourceSettings) -> SourceProto: + + if isinstance(settings, SerialSourceSettings): + from .serial import SerialSource + return await SerialSource.create(device, settings) + if isinstance(settings, HttpSourceSettings): + from .http import HttpSource + return await HttpSource.create(device, settings) + + msg = 'Unknown source input type' + raise TypeError(msg) diff --git a/src/sml2mqtt/sml_value/__init__.py b/src/sml2mqtt/sml_value/__init__.py index 891458c..c793277 100644 --- a/src/sml2mqtt/sml_value/__init__.py +++ b/src/sml2mqtt/sml_value/__init__.py @@ -1,6 +1,2 @@ -from . import __types__, filter - -# isort: split - -from sml2mqtt.sml_value.enum_builder import filter_from_config, transform_from_config, workaround_from_config -from sml2mqtt.sml_value.sml_value import SmlValue +from .sml_value import SmlValue +from .sml_values import SmlValues diff --git a/src/sml2mqtt/sml_value/__types__.py b/src/sml2mqtt/sml_value/__types__.py deleted file mode 100644 index 7ff8c83..0000000 --- a/src/sml2mqtt/sml_value/__types__.py +++ /dev/null @@ -1,5 +0,0 @@ -from typing import Union - -VALUE_TYPE = Union[None, int, float] - -WORKAROUND_TYPE = Union[bool, int, float, str] diff --git a/src/sml2mqtt/sml_value/base.py b/src/sml2mqtt/sml_value/base.py new file mode 100644 index 0000000..f2672e1 --- /dev/null +++ b/src/sml2mqtt/sml_value/base.py @@ -0,0 +1,71 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Final + + +if TYPE_CHECKING: + from collections.abc import Generator + + from smllib.sml import SmlListEntry + + from sml2mqtt.const import SmlFrameValues + + +class SmlValueInfo: + __slots__ = ('value', 'frame', 'last_pub') + + def __init__(self, sml: SmlListEntry, frame: SmlFrameValues, last_pub: float): + self.value: Final = sml + self.frame: Final = frame + + self.last_pub: Final = last_pub + + def __repr__(self): + return f'<{self.__class__.__name__} obis={self.value.obis}>' + + +class ValueOperationBase: + def process_value(self, value: float | None, info: SmlValueInfo) -> float | None: + raise NotImplementedError() + + def describe(self, indent: str = '') -> Generator[str, None, None]: + raise NotImplementedError() + + +class OperationContainerBase: + def __init__(self) -> None: + self.operations: tuple[ValueOperationBase, ...] = () + + def add_operation(self, operation: ValueOperationBase): + self.operations = (*self.operations, operation) + return self + + def insert_operation(self, operation: ValueOperationBase): + self.operations = (operation, *self.operations) + return self + + +class ValueOperationWithStartupBase(ValueOperationBase): + _PROCESS_VALUE_BACKUP_ATTR: Final = '_process_value_original' + + def on_first_value(self, value: float, info: SmlValueInfo): + raise NotImplementedError() + + def enable_on_first_value(self) -> None: + name: Final = self._PROCESS_VALUE_BACKUP_ATTR + if hasattr(self, name): + raise ValueError() + + setattr(self, name, self.process_value) + self.process_value = self._process_value_first + + def _process_value_first(self, value: float | None, info: SmlValueInfo) -> float | None: + if value is None: + return None + + # restore original function + name: Final = self._PROCESS_VALUE_BACKUP_ATTR + self.process_value = getattr(self, name) + delattr(self, name) + + return self.on_first_value(value, info) diff --git a/src/sml2mqtt/sml_value/enum_builder.py b/src/sml2mqtt/sml_value/enum_builder.py deleted file mode 100644 index 1f354ef..0000000 --- a/src/sml2mqtt/sml_value/enum_builder.py +++ /dev/null @@ -1,69 +0,0 @@ -from typing import Any, Dict, List, Type, TypeVar, Union - -import sml2mqtt -from sml2mqtt.config.device import FilterOptionEnum, TransformOptionEnum, TYPE_SML_VALUE_FILTER_CFG, \ - TYPE_SML_VALUE_TRANSFORM_CFG, TYPE_SML_VALUE_WORKAROUND_CFG, WorkaroundOptionEnum -from sml2mqtt.sml_value.filter import ChangeFilter, DiffAbsFilter, \ - DiffFilterBase, DiffPercFilter, FilterBase, RefreshEvery -from sml2mqtt.sml_value.transformations import FactorTransformation, \ - OffsetTransformation, RoundTransformation, TransformationBase -from sml2mqtt.sml_value.workarounds import NegativeOnEnergyMeterStatus, WorkaroundBase - -TYPE_A = TypeVar('TYPE_A') - - -def _from_config(cfg: Union[TYPE_SML_VALUE_FILTER_CFG, TYPE_SML_VALUE_TRANSFORM_CFG], - class_map: Dict[Any, Type[TYPE_A]]) -> List[TYPE_A]: - if cfg is None: - return [] - - ret = [] - for entry in cfg: - for key, cls in class_map.items(): - params = entry.get(key) - if params is not None: - ret.append(cls(params)) - break - else: - raise ValueError(f'Unknown type: {entry}') - return ret - - -def filter_from_config(cfg: TYPE_SML_VALUE_FILTER_CFG) -> List[FilterBase]: - class_dict = { - FilterOptionEnum.diff: DiffAbsFilter, - FilterOptionEnum.perc: DiffPercFilter, - FilterOptionEnum.every: RefreshEvery, - } - filters = _from_config(cfg, class_dict) - - # Default filters - for f in filters: - if isinstance(f, RefreshEvery): - break - else: - filters.append(RefreshEvery(sml2mqtt.CONFIG.general.republish_after)) - - for f in filters: - if isinstance(f, DiffFilterBase): - break - else: - filters.append(ChangeFilter()) - - return filters - - -def transform_from_config(cfg: TYPE_SML_VALUE_TRANSFORM_CFG) -> List[TransformationBase]: - class_dict = { - TransformOptionEnum.factor: FactorTransformation, - TransformOptionEnum.round: RoundTransformation, - TransformOptionEnum.offset: OffsetTransformation, - } - return _from_config(cfg, class_dict) - - -def workaround_from_config(cfg: TYPE_SML_VALUE_WORKAROUND_CFG) -> List[WorkaroundBase]: - class_dict = { - WorkaroundOptionEnum.negative_on_energy_meter_status: NegativeOnEnergyMeterStatus, - } - return _from_config(cfg, class_dict) diff --git a/src/sml2mqtt/sml_value/filter/__init__.py b/src/sml2mqtt/sml_value/filter/__init__.py deleted file mode 100644 index 8febfa5..0000000 --- a/src/sml2mqtt/sml_value/filter/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -from sml2mqtt.sml_value.filter.base import FilterBase - -# isort: split - -from sml2mqtt.sml_value.filter.change import ChangeFilter -from sml2mqtt.sml_value.filter.diff import DiffAbsFilter, DiffFilterBase, DiffPercFilter -from sml2mqtt.sml_value.filter.time import RefreshEvery diff --git a/src/sml2mqtt/sml_value/filter/base.py b/src/sml2mqtt/sml_value/filter/base.py deleted file mode 100644 index a01b8d4..0000000 --- a/src/sml2mqtt/sml_value/filter/base.py +++ /dev/null @@ -1,9 +0,0 @@ -from sml2mqtt.sml_value.__types__ import VALUE_TYPE - - -class FilterBase: - def required(self, value: VALUE_TYPE) -> VALUE_TYPE: - raise NotImplementedError() - - def done(self, value): - raise NotImplementedError() diff --git a/src/sml2mqtt/sml_value/filter/change.py b/src/sml2mqtt/sml_value/filter/change.py deleted file mode 100644 index 689b898..0000000 --- a/src/sml2mqtt/sml_value/filter/change.py +++ /dev/null @@ -1,21 +0,0 @@ -from .base import FilterBase, VALUE_TYPE - - -class ChangeFilter(FilterBase): - def __init__(self): - self.last_value: VALUE_TYPE = None - - def required(self, value: VALUE_TYPE) -> VALUE_TYPE: - if self.last_value is None: - return True - - if value != self.last_value: - return True - - return False - - def done(self, value): - self.last_value = value - - def __repr__(self): - return '' diff --git a/src/sml2mqtt/sml_value/filter/diff.py b/src/sml2mqtt/sml_value/filter/diff.py deleted file mode 100644 index 9afd094..0000000 --- a/src/sml2mqtt/sml_value/filter/diff.py +++ /dev/null @@ -1,48 +0,0 @@ -from typing import Final, Union - -from .base import FilterBase, VALUE_TYPE - - -class DiffFilterBase(FilterBase): - def __init__(self, min_diff: Union[int, float]): - if min_diff < 0: - raise ValueError('min diff must be >= 0') - self.min_diff: Final = min_diff - self.last_value: VALUE_TYPE = None - - def required(self, value: VALUE_TYPE) -> VALUE_TYPE: - if value is None: - return False - if self.last_value is None: - return True - return self.diff(value) - - def done(self, value): - self.last_value = value - - def diff(self, value: Union[int, float]) -> VALUE_TYPE: - raise NotImplementedError() - - -class DiffAbsFilter(DiffFilterBase): - def diff(self, value: Union[int, float]) -> VALUE_TYPE: - if abs(value - self.last_value) < self.min_diff: - return False - return True - - def __repr__(self): - return f'' - - -class DiffPercFilter(DiffFilterBase): - def diff(self, value: Union[int, float]) -> VALUE_TYPE: - if not self.last_value: - return False - - perc = abs(1 - value / self.last_value) * 100 - if perc < self.min_diff: - return False - return True - - def __repr__(self): - return f'' diff --git a/src/sml2mqtt/sml_value/filter/time.py b/src/sml2mqtt/sml_value/filter/time.py deleted file mode 100644 index 7076c5a..0000000 --- a/src/sml2mqtt/sml_value/filter/time.py +++ /dev/null @@ -1,31 +0,0 @@ -from time import monotonic -from typing import Any, Final, Optional - -from .base import FilterBase - - -class RefreshEvery(FilterBase): - def __init__(self, refresh_time: float): - if refresh_time <= 0: - raise ValueError('Refresh time must be > 0') - - self._refresh_time: Final = refresh_time - self._last_refresh: Optional[float] = None - - self._last_value: Any = None - - def required(self, value: Any) -> bool: - if self._last_refresh is None: - return True - - if monotonic() - self._last_refresh >= self._refresh_time: - return True - - return False - - def done(self, value: Any): - self._last_refresh = monotonic() - self._last_value = value - - def __repr__(self): - return f'' diff --git a/src/sml2mqtt/sml_value/operations/__init__.py b/src/sml2mqtt/sml_value/operations/__init__.py new file mode 100644 index 0000000..f2b5694 --- /dev/null +++ b/src/sml2mqtt/sml_value/operations/__init__.py @@ -0,0 +1,13 @@ +from .actions import HeartbeatActionOperation, RefreshActionOperation +from .date_time import DateTimeFinder, MaxValueOperation, MinValueOperation, VirtualMeterOperation +from .filter import ( + DeltaFilterOperation, + OnChangeFilterOperation, + RangeFilterOperation, + SkipZeroMeterOperation, + ThrottleFilterOperation, +) +from .math import FactorOperation, OffsetOperation, RoundOperation +from .operations import OrOperation, SequenceOperation +from .time_series import MaxOfIntervalOperation, MeanOfIntervalOperation, MinOfIntervalOperation +from .workarounds import NegativeOnEnergyMeterWorkaroundOperation diff --git a/src/sml2mqtt/sml_value/operations/_helper.py b/src/sml2mqtt/sml_value/operations/_helper.py new file mode 100644 index 0000000..a21af8f --- /dev/null +++ b/src/sml2mqtt/sml_value/operations/_helper.py @@ -0,0 +1,16 @@ + +def format_period(period: int | float) -> str: + period = period + h = int(period) // 3600 + period %= 3600 + m = int(period) // 60 + s = period % 60 + + parts = [] + for part, unit in ((h, 'hour'), (m, 'minute'), (s, 'second')): + if not part: + continue + if part != 1: + unit += 's' + parts.append(f'{part:d} {unit:s}' if isinstance(part, int) else f'{part:.1f} {unit:s}') + return ' '.join(parts) diff --git a/src/sml2mqtt/sml_value/operations/actions.py b/src/sml2mqtt/sml_value/operations/actions.py new file mode 100644 index 0000000..5122716 --- /dev/null +++ b/src/sml2mqtt/sml_value/operations/actions.py @@ -0,0 +1,61 @@ +from collections.abc import Generator +from time import monotonic +from typing import Final + +from typing_extensions import override + +from sml2mqtt.const import DurationType, get_duration +from sml2mqtt.sml_value.base import SmlValueInfo, ValueOperationBase +from sml2mqtt.sml_value.operations._helper import format_period + + +class RefreshActionOperation(ValueOperationBase): + def __init__(self, every: DurationType): + self.every: Final = get_duration(every) + self.last_time: float = -1 + self.last_value: float | None = None + + @override + def process_value(self, value: float | None, info: SmlValueInfo) -> float | None: + if value is not None: + self.last_value = value + self.last_time = monotonic() + return value + + if monotonic() - self.last_time < self.every: + return None + + self.last_time = monotonic() + return self.last_value + + def __repr__(self): + return f'' + + @override + def describe(self, indent: str = '') -> Generator[str, None, None]: + yield f'{indent:s}- Refresh Action: {format_period(self.every)}' + + +class HeartbeatActionOperation(ValueOperationBase): + def __init__(self, every: DurationType): + self.every: Final = get_duration(every) + self.last_time: float = -1_000_000_000 + self.last_value: float | None = None + + @override + def process_value(self, value: float | None, info: SmlValueInfo) -> float | None: + if value is not None: + self.last_value = value + + if monotonic() - self.last_time < self.every: + return None + + self.last_time = monotonic() + return self.last_value + + def __repr__(self): + return f'' + + @override + def describe(self, indent: str = '') -> Generator[str, None, None]: + yield f'{indent:s}- Heartbeat Action: {format_period(self.every)}' diff --git a/src/sml2mqtt/sml_value/operations/date_time.py b/src/sml2mqtt/sml_value/operations/date_time.py new file mode 100644 index 0000000..0e1313c --- /dev/null +++ b/src/sml2mqtt/sml_value/operations/date_time.py @@ -0,0 +1,153 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Final + +from typing_extensions import override + +from sml2mqtt.const import DateTimeFinder, get_now +from sml2mqtt.sml_value.base import SmlValueInfo, ValueOperationWithStartupBase + + +if TYPE_CHECKING: + from collections.abc import Generator + from datetime import datetime + + +class SupportsDateTimeAction(ValueOperationWithStartupBase): + + def __init__(self, dt_finder: DateTimeFinder, start_now: bool = True): + self._dt_finder: Final = dt_finder + self._next_reset: datetime = dt_finder.get_first_reset(start_now) + + if start_now or not self._dt_finder.enabled: + self.enable_on_first_value() + + def after_next_reset(self, update: bool = True) -> bool: + if not self._dt_finder.enabled: + return False + + if (now := get_now()) >= self._next_reset: + if update: + self._next_reset = self._dt_finder.calc_next(now) + return True + + return False + + @override + def describe(self, indent: str = '') -> Generator[str, None, None]: + if not self._dt_finder.enabled: + yield f'{indent:s} No resets' + return None + + yield f'{indent:s} Next resets:' + yield f'{indent:s} - {self._next_reset if self._next_reset >= get_now() else "now"}' + + next_dt = self._next_reset + + # for every condition we want to show two values + samples = max(self._dt_finder.condition_count * 2 - 1, 2) + for _ in range(samples): + next_dt = self._dt_finder.calc_next(next_dt) + yield f'{indent:s} - {next_dt}' + + +class VirtualMeterOperation(SupportsDateTimeAction): + def __init__(self, dt_finder: DateTimeFinder, start_now: bool): + super().__init__(dt_finder, start_now) + self.last_value: float | None = None + self.offset: float | None = None + + @override + def on_first_value(self, value, info: SmlValueInfo): + self.last_value = value + self.offset = value + return self.process_value(value, info) + + @override + def process_value(self, value: float | None, info: SmlValueInfo) -> float | None: + if value is None: + return None + + if self.after_next_reset(): + self.offset = self.last_value + + self.last_value = value + + if (offset := self.offset) is None: + return None + + return value - offset + + def __repr__(self): + return (f'') + + @override + def describe(self, indent: str = '') -> Generator[str, None, None]: + yield f'{indent:s}- Virtual Meter:' + yield f'{indent:s} Offset: {self.offset}' + yield from super().describe(indent) + + +class MaxValueOperation(SupportsDateTimeAction): + def __init__(self, dt_finder: DateTimeFinder, start_now: bool): + super().__init__(dt_finder, start_now) + self.max_value: float | None = None + + @override + def on_first_value(self, value, info: SmlValueInfo): + self.max_value = value + return self.process_value(value, info) + + @override + def process_value(self, value: float | None, info: SmlValueInfo) -> float | None: + if value is None: + return None + + if self.after_next_reset(): + self.max_value = value + return value + + if self.max_value is None or value <= self.max_value: + return None + + self.max_value = value + return value + + @override + def describe(self, indent: str = '') -> Generator[str, None, None]: + yield f'{indent:s}- Max Value:' + yield f'{indent:s} max: {self.max_value}' + yield from super().describe(indent) + + +class MinValueOperation(SupportsDateTimeAction): + def __init__(self, dt_finder: DateTimeFinder, start_now: bool): + super().__init__(dt_finder, start_now) + self.min_value: float | None = None + + @override + def on_first_value(self, value, info: SmlValueInfo): + self.min_value = value + return self.process_value(value, info) + + @override + def process_value(self, value: float | None, info: SmlValueInfo) -> float | None: + if value is None: + return None + + if self.after_next_reset(): + self.min_value = value + return value + + if self.min_value is None or value >= self.min_value: + return None + + self.min_value = value + return value + + @override + def describe(self, indent: str = '') -> Generator[str, None, None]: + yield f'{indent:s}- Min Value:' + yield f'{indent:s} min: {self.min_value}' + yield from super().describe(indent) diff --git a/src/sml2mqtt/sml_value/operations/filter.py b/src/sml2mqtt/sml_value/operations/filter.py new file mode 100644 index 0000000..dce38c1 --- /dev/null +++ b/src/sml2mqtt/sml_value/operations/filter.py @@ -0,0 +1,148 @@ +from collections.abc import Generator +from time import monotonic +from typing import Final + +from typing_extensions import override + +from sml2mqtt.const import DurationType, get_duration +from sml2mqtt.sml_value.base import SmlValueInfo, ValueOperationBase +from sml2mqtt.sml_value.operations._helper import format_period + + +class OnChangeFilterOperation(ValueOperationBase): + def __init__(self) -> None: + self.last_value: int | float | str | None = None + + @override + def process_value(self, value: float | None, info: SmlValueInfo) -> float | None: + if value is None: + return None + + if self.last_value == value: + return None + + self.last_value = value + return value + + def __repr__(self): + return f'' + + @override + def describe(self, indent: str = '') -> Generator[str, None, None]: + yield f'{indent:s}- On Change Filter' + + +class RangeFilterOperation(ValueOperationBase): + # noinspection PyShadowingBuiltins + def __init__(self, min_value: float | None, max_value: float | None, limit_values: bool = True): + self.min_value: Final = min_value + self.max_value: Final = max_value + self.limit_values: Final = limit_values + + @override + def process_value(self, value: float | None, info: SmlValueInfo) -> float | None: + if value is None: + return None + + if (min_value := self.min_value) is not None and value < min_value: + return min_value if self.limit_values else None + + if (max_value := self.max_value) is not None and value > max_value: + return max_value if self.limit_values else None + + return value + + def __repr__(self): + return (f'') + + @override + def describe(self, indent: str = '') -> Generator[str, None, None]: + yield f'{indent:s}- Range Filter:' + if self.min_value is not None: + yield f'{indent:s} min: {self.min_value}' + if self.max_value is not None: + yield f'{indent:s} max: {self.max_value}' + yield f'{indent:s} limit to min/max: {self.limit_values}' + + +class DeltaFilterOperation(ValueOperationBase): + def __init__(self, min_value: int | float | None = None, min_percent: int | float | None = None): + self.min_value: Final = min_value + self.min_percent: Final = min_percent + + self.last_value: int | float = -1_000_000_000 + + @override + def process_value(self, value: float | None, info: SmlValueInfo) -> float | None: + if value is None: + return None + + last_value = self.last_value + + diff = abs(value - last_value) + + if (delta_min := self.min_value) is not None and diff < delta_min: + return None + + if (min_percent := self.min_percent) is not None: # noqa: SIM102 + # if last value == 0 the percentual change is infinite and we always pass + if last_value != 0: + percent = abs(diff / last_value) * 100 + if percent < min_percent: + return None + + self.last_value = value + return value + + def __repr__(self): + return f'' + + @override + def describe(self, indent: str = '') -> Generator[str, None, None]: + yield f'{indent:s}- Delta Filter:' + if self.min_value: + yield f'{indent:s} Min : {self.min_value}' + if self.min_percent: + yield f'{indent:s} Min %: {self.min_percent}' + + +class SkipZeroMeterOperation(ValueOperationBase): + + @override + def process_value(self, value: float | None, info: SmlValueInfo) -> float | None: + if value is None or value < 0.1: + return None + return value + + def __repr__(self): + return f'' + + @override + def describe(self, indent: str = '') -> Generator[str, None, None]: + yield f'{indent:s}- Zero Meter Filter' + + +class ThrottleFilterOperation(ValueOperationBase): + def __init__(self, period: DurationType): + self.period: Final = get_duration(period) + self.last_time: float = -1_000_000_000 + + @override + def process_value(self, value: float | None, info: SmlValueInfo) -> float | None: + if value is None: + return None + + now = monotonic() + if self.last_time + self.period > now: + return None + + self.last_time = now + return value + + def __repr__(self): + return f'' + + @override + def describe(self, indent: str = '') -> Generator[str, None, None]: + yield f'{indent:s}- Throttle Filter: {format_period(self.period)}' diff --git a/src/sml2mqtt/sml_value/operations/math.py b/src/sml2mqtt/sml_value/operations/math.py new file mode 100644 index 0000000..d2e53f7 --- /dev/null +++ b/src/sml2mqtt/sml_value/operations/math.py @@ -0,0 +1,63 @@ +from collections.abc import Generator +from typing import Final + +from typing_extensions import override + +from sml2mqtt.sml_value.base import SmlValueInfo, ValueOperationBase + + +class FactorOperation(ValueOperationBase): + def __init__(self, factor: int | float): + self.factor: Final = factor + + @override + def process_value(self, value: float | None, info: SmlValueInfo) -> float | None: + if value is None: + return None + return value * self.factor + + def __repr__(self): + return f'' + + @override + def describe(self, indent: str = '') -> Generator[str, None, None]: + yield f'{indent:s}- Factor: {self.factor}' + + +class OffsetOperation(ValueOperationBase): + def __init__(self, offset: int | float): + self.offset: Final = offset + + @override + def process_value(self, value: float | None, info: SmlValueInfo) -> float | None: + if value is None: + return None + return value + self.offset + + def __repr__(self): + return f'' + + @override + def describe(self, indent: str = '') -> Generator[str, None, None]: + yield f'{indent:s}- Offset: {self.offset}' + + +class RoundOperation(ValueOperationBase): + def __init__(self, digits: int): + self.digits: Final = digits if digits else None + + @override + def process_value(self, value: float | None, info: SmlValueInfo) -> float | None: + if value is None: + return None + + if isinstance(value, int): + return value + return round(value, self.digits) + + def __repr__(self): + return f'' + + @override + def describe(self, indent: str = '') -> Generator[str, None, None]: + yield f'{indent:s}- Round: {self.digits if self.digits is not None else "integer"}' diff --git a/src/sml2mqtt/sml_value/operations/operations.py b/src/sml2mqtt/sml_value/operations/operations.py new file mode 100644 index 0000000..09ffacd --- /dev/null +++ b/src/sml2mqtt/sml_value/operations/operations.py @@ -0,0 +1,43 @@ +from collections.abc import Generator + +from typing_extensions import override + +from sml2mqtt.sml_value.base import OperationContainerBase, SmlValueInfo, ValueOperationBase + + +class OrOperation(ValueOperationBase, OperationContainerBase): + + @override + def process_value(self, value: float | None, info: SmlValueInfo) -> float | None: + ret: float | None = None + for op in self.operations: + if (call := op.process_value(value, info)) is not None and ret is None: + ret = call + + return ret + + def __repr__(self): + return f'' + + @override + def describe(self, indent: str = '') -> Generator[str, None, None]: + yield f'{indent:s}- Or:' + for o in self.operations: + yield from o.describe(indent + ' ') + + +class SequenceOperation(ValueOperationBase, OperationContainerBase): + @override + def process_value(self, value: float | None, info: SmlValueInfo) -> float | None: + for op in self.operations: + value = op.process_value(value, info) + return value + + def __repr__(self): + return f'' + + @override + def describe(self, indent: str = '') -> Generator[str, None, None]: + yield f'{indent:s}- Sequence:' + for o in self.operations: + yield from o.describe(indent + ' ') diff --git a/src/sml2mqtt/sml_value/operations/time_series.py b/src/sml2mqtt/sml_value/operations/time_series.py new file mode 100644 index 0000000..4709d29 --- /dev/null +++ b/src/sml2mqtt/sml_value/operations/time_series.py @@ -0,0 +1,113 @@ +from collections.abc import Generator, Sequence +from typing import Final + +from typing_extensions import override + +from sml2mqtt.const import TimeSeries +from sml2mqtt.sml_value.base import SmlValueInfo, ValueOperationBase +from sml2mqtt.sml_value.operations._helper import format_period + + +class TimeSeriesOperationBaseBase(ValueOperationBase): + def __init__(self, time_series: TimeSeries, reset_after_value: bool): + self.time_series: Final = time_series + self.reset_after_value: Final = reset_after_value + + # Makes only sense in combination + if reset_after_value and not time_series.wait_for_data: + raise ValueError() + + @override + def describe(self, indent: str = '') -> Generator[str, None, None]: + yield f'{indent:s} Interval: {format_period(self.time_series.period)}' + yield f'{indent:s} Wait for data: {self.time_series.wait_for_data}' + yield f'{indent:s} Reset after value: {self.reset_after_value}' + + +class TimeSeriesOperationBase(TimeSeriesOperationBaseBase): + def on_values(self, obj: Sequence[float]) -> float | None: + raise NotImplementedError() + + @override + def process_value(self, value: float | None, info: SmlValueInfo) -> float | None: + + ts = info.frame.timestamp + self.time_series.add_value(value, ts) + + if (values := self.time_series.get_values()) is None: + return None + + if self.reset_after_value: + self.time_series.clear() + return self.on_values(values) + + +class TimeDurationSeriesOperationBase(TimeSeriesOperationBaseBase): + def on_values(self, obj: Sequence[tuple[float, float]]) -> float | None: + raise NotImplementedError() + + @override + def process_value(self, value: float | None, info: SmlValueInfo) -> float | None: + + ts = info.frame.timestamp + self.time_series.add_value(value, ts) + + if (values := self.time_series.get_value_duration(ts)) is None: + return None + + if self.reset_after_value: + self.time_series.clear() + return self.on_values(values) + + +class MaxOfIntervalOperation(TimeSeriesOperationBase): + + @override + def on_values(self, obj: Sequence[float]) -> float | None: + return max(obj) + + def __repr__(self): + return f'' + + @override + def describe(self, indent: str = '') -> Generator[str, None, None]: + yield f'{indent:s}- Max Of Interval:' + yield from super().describe(indent) + + +class MinOfIntervalOperation(TimeSeriesOperationBase): + + @override + def on_values(self, obj: Sequence[float]) -> float | None: + return min(obj) + + def __repr__(self): + return f'' + + @override + def describe(self, indent: str = '') -> Generator[str, None, None]: + yield f'{indent:s}- Min Of Interval:' + yield from super().describe(indent) + + +class MeanOfIntervalOperation(TimeDurationSeriesOperationBase): + + @override + def on_values(self, obj: Sequence[tuple[float, float]]) -> float | None: + time = 0.0 + mean = 0.0 + for value, duration in obj: + mean += value * duration + time += duration + + if time <= 0: + return None + return mean / time + + def __repr__(self): + return f'' + + @override + def describe(self, indent: str = '') -> Generator[str, None, None]: + yield f'{indent:s}- Mean Of Interval:' + yield from super().describe(indent) diff --git a/src/sml2mqtt/sml_value/operations/workarounds.py b/src/sml2mqtt/sml_value/operations/workarounds.py new file mode 100644 index 0000000..a2996ad --- /dev/null +++ b/src/sml2mqtt/sml_value/operations/workarounds.py @@ -0,0 +1,38 @@ +from collections.abc import Generator +from typing import Final + +from typing_extensions import override + +from sml2mqtt.errors import RequiredObisValueNotInFrameError +from sml2mqtt.sml_value.base import SmlValueInfo, ValueOperationBase + + +class NegativeOnEnergyMeterWorkaroundOperation(ValueOperationBase): + def __init__(self, meter_obis: str | None = None): + self.meter_obis: Final[str] = '0100010800ff' if meter_obis is None else meter_obis + + @override + def process_value(self, value: float | None, info: SmlValueInfo) -> float | None: + if value is None: + return None + + if (meter := info.frame.get_value(self.meter_obis)) is None: + raise RequiredObisValueNotInFrameError(self.meter_obis) + + status = meter.status + if not isinstance(status, int): + msg = f'Energy Meter status {self.meter_obis:s} is not a valid int: {status} ({type(status)})' + raise TypeError(msg) + + negative = status & 0x20 + if negative: + value *= -1 + + return value + + def __repr__(self): + return f'' + + @override + def describe(self, indent: str = '') -> Generator[str, None, None]: + yield f'{indent:s}- Negative On Status Of Energy Meter {self.meter_obis:s}' diff --git a/src/sml2mqtt/sml_value/setup_operations.py b/src/sml2mqtt/sml_value/setup_operations.py new file mode 100644 index 0000000..1e98051 --- /dev/null +++ b/src/sml2mqtt/sml_value/setup_operations.py @@ -0,0 +1,126 @@ +from collections.abc import Callable +from typing import Protocol + +from pydantic import BaseModel + +from sml2mqtt.config.operations import ( + DeltaFilter, + Factor, + HeartbeatAction, + MaxOfInterval, + MaxValue, + MeanOfInterval, + MinOfInterval, + MinValue, + NegativeOnEnergyMeterWorkaround, + Offset, + OnChangeFilter, + OperationsType, + Or, + RangeFilter, + RefreshAction, + Round, + Sequence, + ThrottleFilter, + VirtualMeter, +) +from sml2mqtt.sml_value.base import OperationContainerBase, ValueOperationBase +from sml2mqtt.sml_value.operations import ( + DeltaFilterOperation, + FactorOperation, + HeartbeatActionOperation, + MaxOfIntervalOperation, + MaxValueOperation, + MeanOfIntervalOperation, + MinOfIntervalOperation, + MinValueOperation, + NegativeOnEnergyMeterWorkaroundOperation, + OffsetOperation, + OnChangeFilterOperation, + OrOperation, + RangeFilterOperation, + RefreshActionOperation, + RoundOperation, + SequenceOperation, + ThrottleFilterOperation, + VirtualMeterOperation, +) + + +def create_workaround_negative_on_energy_meter(enabled_or_obis: bool | str): + if isinstance(enabled_or_obis, str): + return NegativeOnEnergyMeterWorkaroundOperation(meter_obis=enabled_or_obis) + if enabled_or_obis: + return NegativeOnEnergyMeterWorkaroundOperation() + return None + + +def create_or(operations: list[OperationsType]): + return OrOperation() + + +def create_sequence(operations: list[OperationsType]): + return SequenceOperation() + + +MAPPING = { + OnChangeFilter: OnChangeFilterOperation, + HeartbeatAction: HeartbeatActionOperation, + DeltaFilter: DeltaFilterOperation, + + RefreshAction: RefreshActionOperation, + ThrottleFilter: ThrottleFilterOperation, + + Factor: FactorOperation, + Offset: OffsetOperation, + Round: RoundOperation, + RangeFilter: RangeFilterOperation, + + NegativeOnEnergyMeterWorkaround: create_workaround_negative_on_energy_meter, + + Or: create_or, + Sequence: create_sequence, + + VirtualMeter: VirtualMeterOperation, + MinValue: MinValueOperation, + MaxValue: MaxValueOperation, + + MaxOfInterval: MaxOfIntervalOperation, + MinOfInterval: MinOfIntervalOperation, + MeanOfInterval: MeanOfIntervalOperation, +} + + +def get_operation_factory(obj: BaseModel) -> Callable: + for cfg_type, func in MAPPING.items(): + if isinstance(obj, cfg_type): + return func + msg = f'Unknown operation configuration type {type(obj)}' + raise ValueError(msg) + + +def get_kwargs_names(obj: BaseModel) -> list[str]: + return [n for n in dir(obj) if n.startswith('get_kwargs_')] + + +class _HasOperationsProto(Protocol): + operations: list[BaseModel] + + +def setup_operations(parent: OperationContainerBase, cfg_parent: _HasOperationsProto): + for cfg in cfg_parent.operations: + factory = get_operation_factory(cfg) + + if kwarg_names := get_kwargs_names(cfg): + kwargs = {name: value for kwarg_name in kwarg_names for name, value in getattr(cfg, kwarg_name)().items()} + else: + kwargs = cfg.model_dump(exclude={'type'}) + + if (operation_obj := factory(**kwargs)) is None: + continue + + assert isinstance(operation_obj, ValueOperationBase) + parent.add_operation(operation_obj) + + if isinstance(operation_obj, (OrOperation, SequenceOperation)): + setup_operations(operation_obj, cfg) diff --git a/src/sml2mqtt/sml_value/sml_value.py b/src/sml2mqtt/sml_value/sml_value.py index 31d192d..656f1c2 100644 --- a/src/sml2mqtt/sml_value/sml_value.py +++ b/src/sml2mqtt/sml_value/sml_value.py @@ -1,79 +1,46 @@ -from typing import Dict, Final, Iterable, Optional, Union - -from smllib.sml import SmlListEntry +from collections.abc import Generator +from time import monotonic +from typing import Final +from sml2mqtt.const import SmlFrameValues from sml2mqtt.mqtt import MqttObj -from sml2mqtt.sml_value.filter import FilterBase -from sml2mqtt.sml_value.transformations import TransformationBase -from sml2mqtt.sml_value.workarounds import WorkaroundBase +from sml2mqtt.sml_value.base import OperationContainerBase, SmlValueInfo, ValueOperationBase -class SmlValue: - def __init__(self, device: str, obis: str, mqtt: MqttObj, - workarounds: Iterable[WorkaroundBase], - transformations: Iterable[TransformationBase], - filters: Iterable[FilterBase]): +class SmlValue(OperationContainerBase): + def __init__(self, obis: str, mqtt: MqttObj): + super().__init__() - self.device_id: Final = device self.obis: Final = obis self.mqtt: Final = mqtt - self.workarounds: Final = workarounds - self.transformations: Final = transformations - self.filters: Final = filters - - self.sml_value: Optional[SmlListEntry] = None - self.last_value: Union[None, int, float, str] = None + self.last_publish: float = 0 - def set_value(self, sml_value: Optional[SmlListEntry], frame_values: Dict[str, SmlListEntry]): - self.sml_value = sml_value + def __repr__(self): + return f'<{self.__class__.__name__} obis={self.obis} at 0x{id(self):x}>' - # apply all workarounds - for workaround in self.workarounds: - if workaround.enabled: - sml_value = workaround.fix(sml_value, frame_values) + def process_frame(self, frame: SmlFrameValues): + if (sml_value := frame.get_value(self.obis)) is None: + return None - # transform the values - value = None if sml_value is None else sml_value.get_value() - for f in self.transformations: - value = f.process(value) - self.last_value = value + info = SmlValueInfo(sml_value, frame, self.last_publish) + value = sml_value.get_value() - # check if we want to publish - do_publish = False - for refresh in self.filters: - do_publish = refresh.required(value) or do_publish + for op in self.operations: + value = op.process_value(value, info) - if not do_publish: + if value is None: return None self.mqtt.publish(value) - for refresh in self.filters: - refresh.done(value) - - def describe(self, indent=0, indent_width=2) -> str: - - base = " " * indent - once = " " * (indent + indent_width) - twice = " " * (indent + indent_width * 2) - - txt = f'{base}{self.mqtt.topic} ({self.obis}):\n' \ - f'{once}raw value: {self.sml_value.get_value()}\n' \ - f'{once}pub value: {self.last_value}\n' - - if self.workarounds: - txt += f'{once}workarounds:\n' - for w in self.workarounds: - txt += f'{twice}- {w}\n' - - if self.transformations: - txt += f'{once}transformations:\n' - for t in self.transformations: - txt += f'{twice}- {t}\n' - - if self.filters: - txt += f'{once}filters:\n' - for f in self.filters: - txt += f'{twice}- {f}\n' - - return txt + self.last_publish = monotonic() + return value + + def describe(self) -> Generator[str, None, None]: + yield f'' + yield f' obis : {self.obis:s}' + yield f' topic: {self.mqtt.topic:s}' + yield f' operations:' + for op in self.operations: + yield from op.describe(f' ') + yield '' diff --git a/src/sml2mqtt/sml_value/sml_values.py b/src/sml2mqtt/sml_value/sml_values.py new file mode 100644 index 0000000..fbf2676 --- /dev/null +++ b/src/sml2mqtt/sml_value/sml_values.py @@ -0,0 +1,59 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from sml2mqtt.errors import RequiredObisValueNotInFrameError, UnprocessedObisValuesReceivedError + + +if TYPE_CHECKING: + from collections.abc import Generator + + from sml2mqtt.const import SmlFrameValues + from sml2mqtt.sml_value.sml_value import SmlValue + + +class SmlValues: + def __init__(self) -> None: + self._processed_ids: frozenset[str] = frozenset() + self._skipped_ids: frozenset[str] = frozenset() + self._all_ids: frozenset[str] = frozenset() + self._values: tuple[SmlValue, ...] = () + + def __repr__(self): + return ( + f'<{self.__class__.__name__:s} ' + f'processed={",".join(self._processed_ids):s}, ' + f'skipped={",".join(self._skipped_ids):s}>' + ) + + def set_skipped(self, *obis_ids: str): + self._skipped_ids = frozenset(obis_ids) + self._all_ids = self._processed_ids | self._skipped_ids + return self + + def add_value(self, value: SmlValue): + self._processed_ids = self._processed_ids.union((value.obis, )) + self._all_ids = self._processed_ids | self._skipped_ids + self._values = (*self._values, value) + return self + + def process_frame(self, frame: SmlFrameValues): + for value in self._values: + value.process_frame(frame) + + obis_in_frame = frame.obis_ids() + + # Not all obis processed + if obis_left := obis_in_frame - self._all_ids: + entries_left = [frame.get_value(_obis) for _obis in sorted(obis_left)] + raise UnprocessedObisValuesReceivedError(*entries_left) + + # Processed obis not in frame + if obis_missing := self._processed_ids - obis_in_frame: + raise RequiredObisValueNotInFrameError(*sorted(obis_missing)) + + def describe(self) -> Generator[str, None, None]: + yield f'Skipped: {", ".join(sorted(self._skipped_ids))}' + yield '' + for value in self._values: + yield from value.describe() diff --git a/src/sml2mqtt/sml_value/transformations/__init__.py b/src/sml2mqtt/sml_value/transformations/__init__.py deleted file mode 100644 index cb5ccab..0000000 --- a/src/sml2mqtt/sml_value/transformations/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -from sml2mqtt.sml_value.transformations.base import TransformationBase - -# isort: split - -from sml2mqtt.sml_value.transformations.math import FactorTransformation, OffsetTransformation, RoundTransformation diff --git a/src/sml2mqtt/sml_value/transformations/base.py b/src/sml2mqtt/sml_value/transformations/base.py deleted file mode 100644 index f5d89fa..0000000 --- a/src/sml2mqtt/sml_value/transformations/base.py +++ /dev/null @@ -1,6 +0,0 @@ -from sml2mqtt.sml_value.__types__ import VALUE_TYPE - - -class TransformationBase: - def process(self, value: VALUE_TYPE) -> VALUE_TYPE: - raise NotImplementedError() diff --git a/src/sml2mqtt/sml_value/transformations/math.py b/src/sml2mqtt/sml_value/transformations/math.py deleted file mode 100644 index 4d68655..0000000 --- a/src/sml2mqtt/sml_value/transformations/math.py +++ /dev/null @@ -1,44 +0,0 @@ -from typing import Final, Union - -from .base import TransformationBase, VALUE_TYPE - - -class FactorTransformation(TransformationBase): - def __init__(self, factor: Union[int, float]): - self.factor: Final = factor - - def process(self, value: VALUE_TYPE) -> VALUE_TYPE: - if value is None: - return None - return value * self.factor - - def __repr__(self): - return f'' - - -class OffsetTransformation(TransformationBase): - def __init__(self, offset: Union[int, float]): - self.offset: Final = offset - - def process(self, value: VALUE_TYPE) -> VALUE_TYPE: - if value is None: - return None - return value + self.offset - - def __repr__(self): - return f'' - - -class RoundTransformation(TransformationBase): - def __init__(self, digits: int): - self.digits: Final = digits if digits else None - - def process(self, value: VALUE_TYPE) -> VALUE_TYPE: - if value is None: - return None - if isinstance(value, int): - return value - return round(value, self.digits) - - def __repr__(self): - return f'' diff --git a/src/sml2mqtt/sml_value/workarounds/__init__.py b/src/sml2mqtt/sml_value/workarounds/__init__.py deleted file mode 100644 index 41e9ebf..0000000 --- a/src/sml2mqtt/sml_value/workarounds/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -from sml2mqtt.sml_value.workarounds.base import WorkaroundBase - -# isort: split - -from sml2mqtt.sml_value.workarounds.negative_on_energy_status import NegativeOnEnergyMeterStatus diff --git a/src/sml2mqtt/sml_value/workarounds/base.py b/src/sml2mqtt/sml_value/workarounds/base.py deleted file mode 100644 index 544c15e..0000000 --- a/src/sml2mqtt/sml_value/workarounds/base.py +++ /dev/null @@ -1,18 +0,0 @@ -from typing import Dict - -from smllib.sml import SmlListEntry - -from sml2mqtt.sml_value.__types__ import WORKAROUND_TYPE - - -class WorkaroundBase: - def __init__(self, arg: WORKAROUND_TYPE): - self.enabled = True - if arg is False: - self.enabled = False - - def fix(self, value: SmlListEntry, frame_values: Dict[str, SmlListEntry]) -> SmlListEntry: - raise NotImplementedError() - - def __repr__(self): - return f'<{self.__class__.__name__}>' diff --git a/src/sml2mqtt/sml_value/workarounds/negative_on_energy_status.py b/src/sml2mqtt/sml_value/workarounds/negative_on_energy_status.py deleted file mode 100644 index 93efd8d..0000000 --- a/src/sml2mqtt/sml_value/workarounds/negative_on_energy_status.py +++ /dev/null @@ -1,29 +0,0 @@ -from typing import Dict - -from smllib.sml import SmlListEntry - -from .base import WORKAROUND_TYPE, WorkaroundBase - - -class NegativeOnEnergyMeterStatus(WorkaroundBase): - def __init__(self, arg: WORKAROUND_TYPE): - super().__init__(arg) - - self.meter_obis = '0100010800ff' - if isinstance(arg, str): - self.meter_obis = arg - - def fix(self, value: SmlListEntry, frame_values: Dict[str, SmlListEntry]) -> SmlListEntry: - meter = frame_values.get(self.meter_obis) - if meter is None: - raise ValueError(f'Configured meter obis "{self.meter_obis:s}" not found in current frame') - - status = meter.status - if not isinstance(status, int): - raise ValueError(f'Energy Meter status is not a valid int: {status} ({type(status)})') - - negative = status & 0x20 - if negative: - value.value *= -1 - - return value diff --git a/tests/config/test_config.py b/tests/config/test_config.py index 7d235a4..fca1809 100644 --- a/tests/config/test_config.py +++ b/tests/config/test_config.py @@ -1,15 +1,39 @@ import pytest from pydantic import ValidationError +from sml2mqtt.config import cleanup_validation_errors from sml2mqtt.config.config import SmlValueConfig def test_err_msg(): with pytest.raises(ValidationError) as e: - SmlValueConfig(transformations=[{'factor': 1, 'offset': 2}]) + SmlValueConfig.model_validate({ + 'obis': '00112233445566', + 'mqtt': {'topic': 'OBIS'}, + 'operations': [{'factor': 1, 'offset': 2}] + }) assert str(e.value) == \ '1 validation error for SmlValueConfig\n' \ - 'transformations\n' \ - ' Only one entry allowed! Got 2: factor, offset (type=value_error)' + 'operations.0\n' \ + " Invalid key names [type=invalid_key_names, input_value={'factor': 1, 'offset': 2}, input_type=dict]" + + +def test_error_message(): + + with pytest.raises(ValidationError) as e: + SmlValueConfig.model_validate({ + 'obis': '00112233445566', + 'mqtt': {'topic': 'OBIS'}, + 'operations': [ + {'negative on energy meter status': True}, + {'factor': 3}, {'offset': 100}, {'round': 2}, + {'or': [{'change filvter': True}, {'heartbeat action': 120}]} + ] + }) + + assert '\n' + cleanup_validation_errors(str(e.value)) == ''' +1 validation error for SmlValueConfig +operations.4.or.0 + Invalid key names [type=invalid_key_names, input_value={'change filvter': True}, input_type=dict]''' diff --git a/tests/config/test_default.py b/tests/config/test_default.py index 36c6355..f41f933 100644 --- a/tests/config/test_default.py +++ b/tests/config/test_default.py @@ -1,56 +1,59 @@ +import re + from sml2mqtt.config import CONFIG def test_default(): - yaml = CONFIG.generate_default_yaml() + + # Replace dynamically created identifier + yaml = re.sub(r'identifier: sml2mqtt-\w+', 'identifier: sml2mqtt-A1b2', yaml) + assert '\n' + yaml == ''' logging: level: INFO # Log level - file: sml2mqtt.log # Log file path (absolute or relative to config file) + file: sml2mqtt.log # Log file path (absolute or relative to config file) or "stdout" mqtt: connection: - client id: sml2mqtt + identifier: sml2mqtt-A1b2 host: localhost port: 1883 user: '' password: '' - tls: false - tls insecure: false - topic prefix: sml2mqtt + topic prefix: sml2mqtt # Prefix for all topics. Set to empty string to disable defaults: - qos: 0 # Default value for QOS if no other QOS value in the config entry is set + qos: 0 # Default value for QoS if no other QoS value in the config entry is set retain: false # Default value for retain if no other retain value in the config entry is set last will: topic: status # Topic fragment for building this topic with the parent topic general: Wh in kWh: true # Automatically convert Wh to kWh republish after: 120 # Republish automatically after this time (if no other filter configured) -ports: -- url: COM1 # Device path - timeout: 3 # Seconds after which a timeout will be detected (default=3) -- url: /dev/ttyS0 # Device path - timeout: 3 # Seconds after which a timeout will be detected (default=3) +inputs: +- type: serial + url: COM1 # Device path + timeout: 6 # Seconds after which a timeout will be detected (default=6) +- type: serial + url: /dev/ttyS0 # Device path + timeout: 6 # Seconds after which a timeout will be detected (default=6) devices: # Device configuration by ID or url - DEVICE_ID_HEX: + device_id_hex: mqtt: # Optional MQTT configuration for this meter. topic: DEVICE_BASE_TOPIC # Topic fragment for building this topic with the parent topic status: # Optional MQTT status topic configuration for this meter topic: status # Topic fragment for building this topic with the parent topic skip: # OBIS codes (HEX) of values that will not be published (optional) - - OBIS - values: # Special configurations for each of the values (optional) - OBIS: - mqtt: # Mqtt config for this entry (optional) - topic: OBIS # Topic fragment for building this topic with the parent topic - workarounds: # Workarounds for the value (optional) - - negative on energy meter status: true - transformations: # Mathematical transformations for the value (optional) - - factor: 3 - - offset: 100 - - round: 2 - filters: # Refresh options for the value (optional) - - diff: 10 - - perc: 10 - - every: 120 + - '00112233445566' + values: # Configurations for each of the values (optional) + - obis: '00112233445566' # Obis code for this value + mqtt: # Mqtt config for this value (optional) + topic: OBIS # Topic fragment for building this topic with the parent topic + operations: # A sequence of operations that will be evaluated one after another. + # If one operation blocks this will return nothing. + - negative on energy meter status: true # Set to "true" to enable or to "false" to disable workaround. If the default obis code for the energy meter is wrong set to the appropriate meter obis code instead + - factor: 3 # Factor with which the value gets multiplied + - offset: 100 # Offset that gets added on the value + - round: 2 # Round to the specified digits + - type: change filter # Filter which passes only changes + - refresh action: 600 # Refresh interval ''' diff --git a/tests/config/test_types.py b/tests/config/test_types.py new file mode 100644 index 0000000..bd66e71 --- /dev/null +++ b/tests/config/test_types.py @@ -0,0 +1,26 @@ +from __future__ import annotations + +from datetime import timedelta + +from pydantic import BaseModel + +from sml2mqtt.config.types import ObisHex # noqa: TCH001 +from sml2mqtt.const import DurationType # noqa: TCH001 + + +def test_obis(): + class TestObis(BaseModel): + value: ObisHex + + assert TestObis.model_validate({'value': '0100000009ff'}).value == '0100000009ff' + assert TestObis.model_validate({'value': '0100000009FF'}).value == '0100000009ff' + assert TestObis.model_validate({'value': '0100000009FF '}).value == '0100000009ff' + + +def test_duration(): + class TestObis(BaseModel): + value: DurationType + + assert TestObis.model_validate({'value': 0}).value == 0 + assert TestObis.model_validate({'value': 13.5}).value == 13.5 + assert TestObis.model_validate({'value': '01:00'}).value == timedelta(hours=1) diff --git a/tests/conftest.py b/tests/conftest.py index b3cda67..bec6de8 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,17 +1,78 @@ -from binascii import a2b_hex -from unittest.mock import AsyncMock, Mock +import logging +import traceback +from typing import TYPE_CHECKING import pytest -from smllib.reader import SmlFrame - -import sml2mqtt.device.sml_device -import sml2mqtt.device.sml_serial +from helper import PatchedSmlStreamReader +from tests.sml_data import ( + sml_data_1, + sml_data_1_analyze, + sml_frame_1, + sml_frame_1_analyze, + sml_frame_1_values, + sml_frame_2, + sml_frame_2_analyze, + sml_frame_2_values, +) + +import sml2mqtt.const.task as task_module import sml2mqtt.mqtt.mqtt_obj -from sml2mqtt.device import Device, DeviceStatus -from sml2mqtt.mqtt import MqttObj +from sml2mqtt import CMD_ARGS +from sml2mqtt.runtime import shutdown as shutdown_module +from sml2mqtt.sml_device import sml_device as sml_device_module + + +if TYPE_CHECKING: + sml_data_1 = sml_data_1 + sml_data_1_analyze = sml_data_1_analyze + + sml_frame_1 = sml_frame_1 + sml_frame_1_values = sml_frame_1_values + sml_frame_1_analyze = sml_frame_1_analyze + + sml_frame_2 = sml_frame_2 + sml_frame_2_values = sml_frame_2_values + sml_frame_2_analyze = sml_frame_2_analyze + + +class PatchedMonotonic: + def __init__(self): + self._now: int | float = 0 + self._mp = pytest.MonkeyPatch() + + def _get_monotonic(self): + return self._now + + def patch_name(self, target: str): + self._mp.setattr(target, self._get_monotonic) + + def patch(self, target: str | object, name: str | object): + self._mp.setattr(target, name, value=self._get_monotonic) + + def undo(self): + self._mp.undo() + def add(self, secs: float): + self._now += secs -@pytest.fixture + def set(self, secs: float): + self._now = secs + + +@pytest.fixture() +def monotonic(): + p = PatchedMonotonic() + + p.patch_name('sml2mqtt.sml_value.operations.filter.monotonic') + p.patch_name('sml2mqtt.sml_value.operations.actions.monotonic') + + try: + yield p + finally: + p.undo() + + +@pytest.fixture() def no_mqtt(monkeypatch): pub_list = [] @@ -20,126 +81,89 @@ def pub_func(topic: str, value, qos: int, retain: bool): pub_list.append((topic, value, qos, retain)) monkeypatch.setattr(sml2mqtt.mqtt.mqtt_obj, 'pub_func', pub_func) + return pub_list + + +@pytest.fixture() +def stream_reader(monkeypatch): + r = PatchedSmlStreamReader() + monkeypatch.setattr(sml_device_module, 'SmlStreamReader', lambda: r) + return r + + +@pytest.fixture(autouse=True) +def _clear_mqtt(monkeypatch): + monkeypatch.setattr(sml2mqtt.mqtt.BASE_TOPIC, 'children', []) + + +@pytest.fixture(autouse=True) +def check_no_logged_error(caplog, request): + caplog.set_level(logging.DEBUG) + + yield None + + all_levels = set(logging._levelToName) + fail_on_default = {lvl for lvl in all_levels if lvl >= logging.WARNING} + fail_on = fail_on_default.copy() + + markers = request.node.own_markers + for marker in markers: + if marker.name == 'ignore_log_errors': + fail_on.discard(logging.ERROR) + elif marker.name == 'ignore_log_warnings': + fail_on.discard(logging.WARNING) + + msgs = [] + for fail_lvls, phase in ((fail_on_default, 'setup'), (fail_on, 'call'), (fail_on_default, 'teardown')): + for record in caplog.get_records(phase): + if record.levelno not in all_levels: + msg = f'Unknown log level: {record.levelno}! Supported: {", ".join(all_levels)}' + raise ValueError(msg) + + if record.levelno in fail_lvls: + msgs.append(f'{record.name:20s} | {record.levelname:7} | {record.getMessage():s}') + + if msgs: + pytest.fail(reason='Error in log:\n' + '\n'.join(msgs)) + + +@pytest.fixture(autouse=True) +def _wrap_all_tasks(monkeypatch): + + async def wrapped_future(coro): + try: + return await coro + except Exception: + for line in traceback.format_exc().splitlines(): + logging.getLogger('task_wrap').error(line) + raise + + original = task_module.asyncio_create_task + + def create_task(coro, *, name=None): + return original(wrapped_future(coro), name=name) + + monkeypatch.setattr(task_module, 'create_task', create_task) + + +@pytest.fixture() +def arg_analyze(monkeypatch): + monkeypatch.setattr(CMD_ARGS, 'analyze', True) + sml2mqtt.mqtt.patch_analyze() + + yield None + + module = sml2mqtt.mqtt.mqtt_obj + assert hasattr(module, 'pub_func') + module.pub_func = module.publish + + +@pytest.fixture(autouse=True) +async def _patch_shutdown(monkeypatch): + objs = () + monkeypatch.setattr(shutdown_module, 'SHUTDOWN_OBJS', objs) + + yield - yield pub_list - - -@pytest.fixture -def sml_data_1(): - data = b'1B1B1B1B01010101760501188E6162006200726500000101760101070000000000000B000000000000000000000101636877007' \ - b'60501188E626200620072650000070177010B000000000000000000000172620165002EC3F47A77078181C78203FF0101010104' \ - b'45425A0177070100000009FF010101010B000000000000000000000177070100010800FF6401018001621E52FB690000000A7AC' \ - b'1BC170177070100010801FF0101621E52FB690000000A74B1EA770177070100010802FF0101621E52FB6900000000060FD1A001' \ - b'77070100020800FF6401018001621E52FB69000000000D19E1C00177070100100700FF0101621B52FE55000089D901770701002' \ - b'40700FF0101621B52FE55000020220177070100380700FF0101621B52FE5500000A9201770701004C0700FF0101621B52FE5500' \ - b'005F2501010163810200760501188E636200620072650000020171016325FC000000001B1B1B1B1A0356F5' - - yield data - - -@pytest.fixture -def sml_frame_1(): - data = b'760500531efa620062007263010176010105001bb4fe0b0a0149534b0005020de272620165001bb32e620163a71400760500531e' \ - b'fb620062007263070177010b0a0149534b0005020de2070100620affff72620165001bb32e757707010060320101010101010449' \ - b'534b0177070100600100ff010101010b0a0149534b0005020de20177070100010800ff65001c010401621e52ff650026bea90177' \ - b'070100020800ff0101621e52ff62000177070100100700ff0101621b52005301100101016350ba00760500531efc620062007263' \ - b'0201710163ba1900' - - yield SmlFrame(a2b_hex(data)) - - -@pytest.fixture -def sml_frame_2(): - data = b'7605065850a66200620072630101760107ffffffffffff05021d70370b0a014c475a0003403b4972620165021d7707016326de' \ - b'007605065850a762006200726307017707ffffffffffff0b0a014c475a0003403b49070100620affff72620165021d77077577' \ - b'0701006032010101010101044c475a0177070100600100ff010101010b0a014c475a0003403b490177070100010800ff65001c' \ - b'010472620165021d7707621e52ff690000000003152c450177070100020800ff0172620165021d7707621e52ff690000000000' \ - b'0000000177070100100700ff0101621b52005900000000000000fb010101637264007605065850a86200620072630201710163' \ - b'1c8c00' - - yield SmlFrame(a2b_hex(data)) - - -@pytest.fixture -def sml_frame_1_analyze(): - msg = """ -Received Frame - -> b'760500531efa620062007263010176010105001bb4fe0b0a0149534b0005020de272620165001bb32e620163a71400760500531efb620062007263070177010b0a0149534b0005020de2070100620affff72620165001bb32e757707010060320101010101010449534b0177070100600100ff010101010b0a0149534b0005020de20177070100010800ff65001c010401621e52ff650026bea90177070100020800ff0101621e52ff62000177070100100700ff0101621b52005301100101016350ba00760500531efc6200620072630201710163ba1900' - - - transaction_id: 00531efa - group_no : 0 - abort_on_error: 0 - message_body - codepage : None - client_id : None - req_file_id: 001bb4fe - server_id : 0a0149534b0005020de2 - ref_time : 1815342 - sml_version: 1 - crc16 : 42772 - - transaction_id: 00531efb - group_no : 0 - abort_on_error: 0 - message_body - client_id : None - server_id : 0a0149534b0005020de2 - list_name : 0100620affff - act_sensor_time : 1815342 - val_list: - - obis : 010060320101 (1-0:96.50.1*1) - status : None - val_time : None - unit : None - scaler : None - value : ISK - value_signature: None - - obis : 0100600100ff (1-0:96.1.0*255) - status : None - val_time : None - unit : None - scaler : None - value : 0a0149534b0005020de2 - value_signature: None - - obis : 0100010800ff (1-0:1.8.0*255) - status : 1835268 - val_time : None - unit : 30 - scaler : -1 - value : 2539177 - value_signature: None - -> 253917.7Wh (Zählerstand Total) - - obis : 0100020800ff (1-0:2.8.0*255) - status : None - val_time : None - unit : 30 - scaler : -1 - value : 0 - value_signature: None - -> 0.0Wh (Wirkenergie Total) - - obis : 0100100700ff (1-0:16.7.0*255) - status : None - val_time : None - unit : 27 - scaler : 0 - value : 272 - value_signature: None - -> 272W (aktuelle Wirkleistung) - list_signature : None - act_gateway_time: None - crc16 : 20666 - - transaction_id: 00531efc - group_no : 0 - abort_on_error: 0 - message_body - global_signature: None - crc16 : 47641 -""" - return msg + for obj in objs: + await obj.do() diff --git a/tests/const/test_protocols.py b/tests/const/test_protocols.py new file mode 100644 index 0000000..8c87931 --- /dev/null +++ b/tests/const/test_protocols.py @@ -0,0 +1,31 @@ +import inspect +from typing import Protocol + +import pytest + +from sml2mqtt.const import DeviceProto +from sml2mqtt.sml_device import SmlDevice + + +def assert_signatures(a, b): + sig_a = inspect.signature(a) if a is not None else a + sig_b = inspect.signature(b) if b is not None else b + assert sig_a == sig_b, f'\n {sig_a}\n {sig_b}\n' + + +@pytest.mark.parametrize( + ('proto', 'cls'), [(DeviceProto, SmlDevice), ] +) +def test_protocols(proto: type[Protocol], cls: type): + for name, proto_obj in inspect.getmembers(proto): + if name.startswith('_'): + continue + + class_obj = getattr(cls, name) + + if isinstance(class_obj, property): + assert_signatures(proto_obj.fget, class_obj.fget) + assert_signatures(proto_obj.fset, class_obj.fset) + assert_signatures(proto_obj.fdel, class_obj.fdel) + else: + assert_signatures(proto_obj, class_obj) diff --git a/tests/const/test_time_series.py b/tests/const/test_time_series.py new file mode 100644 index 0000000..538c4d4 --- /dev/null +++ b/tests/const/test_time_series.py @@ -0,0 +1,97 @@ +from sml2mqtt.const import TimeSeries + + +def test_time_series_boundaries(): + + for wait_for_data in (True, False): + t = TimeSeries(10, wait_for_data=wait_for_data) + t.add_value(1, 0) + t.add_value(2, 10) + assert list(t.get_values()) == [1, 2] + assert t.get_value_duration(10) == [(1, 10), (2, 0)] + assert t.is_full + + t.add_value(3, 20) + assert list(t.get_values()) == [2, 3] + assert t.get_value_duration(20) == [(2, 10), (3, 0)] + assert t.is_full + + t.add_value(4, 30) + assert list(t.get_values()) == [3, 4] + assert t.get_value_duration(30) == [(3, 10), (4, 0)] + assert t.is_full + + +def test_time_series(): + t = TimeSeries(10) + + t.add_value(1, 3) + t.add_value(2, 10) + assert list(t.get_values()) == [1, 2] + assert t.get_value_duration(10) == [(1, 7), (2, 0)] + + t.add_value(3, 15) + assert list(t.get_values()) == [1, 2, 3] + assert t.get_value_duration(15) == [(1, 5), (2, 5), (3, 0)] + + t.add_value(4, 17) + assert list(t.get_values()) == [1, 2, 3, 4] + assert t.get_value_duration(17) == [(1, 3), (2, 5), (3, 2), (4, 0)] + + t.add_value(5, 20) + assert list(t.get_values()) == [2, 3, 4, 5] + assert t.get_value_duration(20) == [(2, 5), (3, 2), (4, 3), (5, 0)] + + +def test_time_series_start(): + t = TimeSeries(10, wait_for_data=False) + + for _ in range(2): + assert t.get_values() is None + assert t.get_value_duration(0) is None + + t.add_value(1, 3) + assert list(t.get_values()) == [1] + assert t.get_value_duration(3) == [(1, 0)] + + t.add_value(None, 7) + assert t.get_value_duration(7) == [(1, 4)] + + t.add_value(None, 13) + assert t.get_value_duration(13) == [(1, 10)] + + t.add_value(None, 100) + assert t.get_value_duration(100) == [(1, 10)] + + t.add_value(2, 200) + assert list(t.get_values()) == [1, 2] + assert t.get_value_duration(200) == [(1, 10), (2, 0)] + + t.clear() + + +def test_time_series_start_wait_for_data(): + t = TimeSeries(10, wait_for_data=True) + + for _ in range(2): + assert t.get_values() is None + assert t.get_value_duration(0) is None + + t.add_value(1, 3) + assert t.get_values() is None + assert t.get_value_duration(3) is None + + t.add_value(None, 7) + assert t.get_value_duration(7) is None + + t.add_value(None, 13) + assert t.get_value_duration(13) == [(1, 10)] + + t.add_value(None, 100) + assert t.get_value_duration(100) == [(1, 10)] + + t.add_value(2, 200) + assert list(t.get_values()) == [1, 2] + assert t.get_value_duration(200) == [(1, 10), (2, 0)] + + t.clear() diff --git a/tests/device/conftest.py b/tests/device/conftest.py deleted file mode 100644 index fb9ff7d..0000000 --- a/tests/device/conftest.py +++ /dev/null @@ -1,86 +0,0 @@ -from typing import Set, Union -from unittest.mock import AsyncMock, Mock - -import pytest -from smllib import SmlFrame, SmlStreamReader - -import sml2mqtt.device.sml_device -import sml2mqtt.device.sml_serial -from sml2mqtt import CMD_ARGS -from sml2mqtt.config.config import PortSettings -from sml2mqtt.device import Device, DeviceStatus -from sml2mqtt.mqtt import MqttObj, patch_analyze - - -@pytest.fixture() -def no_serial(monkeypatch): - - m = Mock() - m.create = AsyncMock() - - monkeypatch.setattr(sml2mqtt.device, 'SmlSerial', m) - monkeypatch.setattr(sml2mqtt.device.sml_serial, 'SmlSerial', m) - return m - - -@pytest.fixture(autouse=True) -def clean_devices(monkeypatch): - monkeypatch.setattr(sml2mqtt.device.sml_device, 'ALL_DEVICES', {}) - return None - - -class TestingStreamReader: - def __init__(self, reader: SmlStreamReader): - self.reader = reader - self.data = None - - def add(self, data: Union[SmlFrame, bytes]): - if isinstance(data, SmlFrame): - self.data = data - self.reader.clear() - else: - self.data = None - self.reader.add(data) - - def get_frame(self): - if self.data is None: - return self.reader.get_frame() - return self.data - - -class TestingDevice(Device): - - def __init__(self, url: str, timeout: float, skip_values: Set[str], mqtt_device: MqttObj): - super().__init__(url, timeout, skip_values, mqtt_device) - self.stream = TestingStreamReader(self.stream) - - self.testing_raise_on_status = True - - def set_status(self, new_status: DeviceStatus) -> bool: - if new_status is DeviceStatus.ERROR and self.testing_raise_on_status: - raise - return super().set_status(new_status) - - -@pytest.fixture() -async def device(no_serial): - device_url = 'device_url' - - mqtt_base = MqttObj('testing', 0, False).update() - mqtt_device = mqtt_base.create_child(device_url) - - obj = await TestingDevice.create(PortSettings(url=device_url), 1, set(), mqtt_device) - - return obj - - -@pytest.fixture() -def arg_analyze(monkeypatch): - monkeypatch.setattr(CMD_ARGS, 'analyze', True) - patch_analyze() - - yield None - - module = sml2mqtt.mqtt.mqtt_obj - assert hasattr(module, 'pub_func') - module.pub_func = module.publish diff --git a/tests/device/frames/test_frame_1.py b/tests/device/frames/test_frame_1.py deleted file mode 100644 index 887a6ac..0000000 --- a/tests/device/frames/test_frame_1.py +++ /dev/null @@ -1,114 +0,0 @@ -import logging - -from smllib.reader import SmlFrame - -from device.conftest import TestingDevice -from sml2mqtt import CONFIG -from sml2mqtt.config.device import SmlDeviceConfig - - -async def test_frame_no_match_obis_id(device: TestingDevice, no_serial, caplog, monkeypatch, - sml_frame_1: SmlFrame, sml_frame_1_analyze, arg_analyze): - caplog.set_level(logging.DEBUG) - monkeypatch.setattr(CONFIG.general, 'device_id_obis', ['0100000009ff', '01006001ffff']) - - device.testing_raise_on_status = False - device.serial_data_read(sml_frame_1) - - msg = "\n".join(x.msg for x in caplog.records) - - assert msg == sml_frame_1_analyze + """ -Found none of the following obis ids in the sml frame: 0100000009ff, 01006001ffff -Received Frame - -> b'760500531efa620062007263010176010105001bb4fe0b0a0149534b0005020de272620165001bb32e620163a71400760500531efb620062007263070177010b0a0149534b0005020de2070100620affff72620165001bb32e757707010060320101010101010449534b0177070100600100ff010101010b0a0149534b0005020de20177070100010800ff65001c010401621e52ff650026bea90177070100020800ff0101621e52ff62000177070100100700ff0101621b52005301100101016350ba00760500531efc6200620072630201710163ba1900' - -ERROR -testing/device_url/status: ERROR (QOS: 0, retain: False)""" - - -async def test_frame_no_config(device: TestingDevice, no_serial, caplog, monkeypatch, - sml_frame_1: SmlFrame, sml_frame_1_analyze, arg_analyze): - caplog.set_level(logging.DEBUG) - monkeypatch.setattr(CONFIG.general, 'device_id_obis', ['0100600100ff']) - - device.testing_raise_on_status = False - device.serial_data_read(sml_frame_1) - - msg = "\n".join(x.msg for x in caplog.records) - - assert msg == sml_frame_1_analyze + """ -Found obis id 0100600100ff in the sml frame -No configuration found for 0a0149534b0005020de2 -Creating default value handler for 0100010800ff -Creating default value handler for 0100100700ff -Creating default value handler for 010060320101 -testing/0a0149534b0005020de2/010060320101: ISK (QOS: 0, retain: False) -testing/0a0149534b0005020de2/0100010800ff: 253.9177 (QOS: 0, retain: False) -testing/0a0149534b0005020de2/0100100700ff: 272 (QOS: 0, retain: False) -OK -testing/0a0149534b0005020de2/status: OK (QOS: 0, retain: False) - -testing/0a0149534b0005020de2/0100010800ff (0100010800ff): - raw value: 253.9177 - pub value: 253.9177 - filters: - - - - - -testing/0a0149534b0005020de2/0100100700ff (0100100700ff): - raw value: 272 - pub value: 272 - filters: - - - - - -testing/0a0149534b0005020de2/010060320101 (010060320101): - raw value: ISK - pub value: ISK - filters: - - - - - -SHUTDOWN -testing/0a0149534b0005020de2/status: SHUTDOWN (QOS: 0, retain: False)""" - - -async def test_frame_with_config(device: TestingDevice, no_serial, caplog, monkeypatch, - sml_frame_1: SmlFrame, sml_frame_1_analyze, arg_analyze): - caplog.set_level(logging.DEBUG) - - monkeypatch.setattr(CONFIG.general, 'device_id_obis', ['0100600100ff']) - monkeypatch.setitem(CONFIG.devices, '0a0149534b0005020de2', SmlDeviceConfig( - skip=['010060320101'] - )) - - device.serial_data_read(sml_frame_1) - - msg = "\n".join(x.msg for x in caplog.records) - - assert msg == sml_frame_1_analyze + """ -Found obis id 0100600100ff in the sml frame -Configuration found for 0a0149534b0005020de2 -Creating default value handler for 0100010800ff -Creating default value handler for 0100100700ff -testing/0a0149534b0005020de2/0100010800ff: 253.9177 (QOS: 0, retain: False) -testing/0a0149534b0005020de2/0100100700ff: 272 (QOS: 0, retain: False) -OK -testing/0a0149534b0005020de2/status: OK (QOS: 0, retain: False) - -testing/0a0149534b0005020de2/0100010800ff (0100010800ff): - raw value: 253.9177 - pub value: 253.9177 - filters: - - - - - -testing/0a0149534b0005020de2/0100100700ff (0100100700ff): - raw value: 272 - pub value: 272 - filters: - - - - - -SHUTDOWN -testing/0a0149534b0005020de2/status: SHUTDOWN (QOS: 0, retain: False)""" diff --git a/tests/device/frames/test_frame_2.py b/tests/device/frames/test_frame_2.py deleted file mode 100644 index a2714aa..0000000 --- a/tests/device/frames/test_frame_2.py +++ /dev/null @@ -1,27 +0,0 @@ -import asyncio -import logging - -from smllib.reader import SmlFrame - -from sml2mqtt import CONFIG -from sml2mqtt.config.device import SmlDeviceConfig -from sml2mqtt.device import Device - - -async def test_frame_2(device: Device, no_serial, caplog, sml_frame_2: SmlFrame, monkeypatch, no_mqtt): - caplog.set_level(logging.DEBUG) - - monkeypatch.setattr(CONFIG.general, 'device_id_obis', ['0100600100ff']) - monkeypatch.setitem(CONFIG.devices, '0a014c475a0003403b49', SmlDeviceConfig( - mqtt={'topic': 'xxxx'} - )) - - device.process_frame(sml_frame_2) - await asyncio.sleep(0.01) - - assert no_mqtt == [ - ('testing/xxxx/010060320101', 'LGZ', 0, False), - ('testing/xxxx/0100010800ff', 5171.9237, 0, False), - ('testing/xxxx/0100100700ff', 251, 0, False), - ('testing/xxxx/status', 'OK', 0, False) - ] diff --git a/tests/device/test_device.py b/tests/device/test_device.py deleted file mode 100644 index c000f07..0000000 --- a/tests/device/test_device.py +++ /dev/null @@ -1,27 +0,0 @@ -import asyncio -from time import monotonic -from unittest.mock import Mock - -from sml2mqtt.device import Device, SmlSerial -from sml2mqtt.mqtt import MqttObj - - -async def test_device_await(device: Device, no_serial, caplog): - device = Device('test', 1, set(), MqttObj('testing', 0, False)) - device.serial = SmlSerial() - device.serial.url = 'test' - device.serial.transport = Mock() - device.serial.transport.is_closing = lambda: False - device.start() - - async def cancel(): - await asyncio.sleep(0.3) - device.stop() - - t = asyncio.create_task(cancel()) - start = monotonic() - await asyncio.wait_for(device, 1) - await t - assert monotonic() - start < 0.4 - - await asyncio.sleep(0.1) diff --git a/tests/device/test_watchdog.py b/tests/device/test_watchdog.py deleted file mode 100644 index d0853c8..0000000 --- a/tests/device/test_watchdog.py +++ /dev/null @@ -1,76 +0,0 @@ -import asyncio -from binascii import a2b_hex -from unittest.mock import Mock - -from sml2mqtt.config.config import PortSettings -from sml2mqtt.device import Device, DeviceStatus -from sml2mqtt.device.watchdog import Watchdog -from sml2mqtt.mqtt import MqttObj - - -async def test_watchdog_expire(): - - m = Mock() - m.assert_not_called() - - w = Watchdog(0.1, m) - w.start() - - await asyncio.sleep(0.15) - m.assert_called_once() - w.feed() - await asyncio.sleep(0.15) - assert m.call_count == 2 - - w.cancel() - - # Assert that the task is properly canceled - await asyncio.sleep(0.05) - assert w.task is None - - -async def test_watchdog_no_expire(): - - m = Mock() - m.assert_not_called() - - w = Watchdog(0.1, m) - w.start() - for _ in range(4): - w.feed() - await asyncio.sleep(0.06) - - m.assert_not_called() - - w.cancel() - - # Assert that the task is properly canceled - await asyncio.sleep(0.05) - assert w.task is None - - -async def test_watchdog_setup_and_feed(no_serial, sml_data_1): - device_url = 'watchdog_test' - - mqtt_base = MqttObj('testing', 0, False).update() - mqtt_device = mqtt_base.create_child(device_url) - - obj = await Device.create(PortSettings(url=device_url), 0.2, set(), mqtt_device) - obj.start() - assert obj.status == DeviceStatus.STARTUP - - await asyncio.sleep(0.3) - assert obj.status == DeviceStatus.MSG_TIMEOUT - - for _ in range(5): - await asyncio.sleep(0.15) - obj.serial_data_read(a2b_hex(sml_data_1)) - assert obj.status != DeviceStatus.MSG_TIMEOUT - - await asyncio.sleep(0.3) - assert obj.status == DeviceStatus.MSG_TIMEOUT - - async def cancel(): - obj.stop() - - await asyncio.gather(obj, cancel()) diff --git a/tests/helper.py b/tests/helper.py new file mode 100644 index 0000000..33bdd29 --- /dev/null +++ b/tests/helper.py @@ -0,0 +1,68 @@ +from asyncio import sleep +from collections.abc import Callable +from unittest.mock import Mock + +from smllib import SmlStreamReader +from smllib.builder import CTX_HINT +from smllib.errors import CrcError +from typing_extensions import override + +from sml2mqtt.const import EnhancedSmlFrame + + +async def wait_for_call(mock: Mock | Callable, timeout: float) -> Mock: + mock.assert_not_called() + + interval = 0.01 + cycles = int(timeout / interval) + + while not mock.called: + await sleep(interval) + + cycles -= 1 + if cycles <= 0: + raise TimeoutError() + + return mock + + +class PatchedSmlStreamReader(SmlStreamReader): + _CRC_ERROR = 'CRC_ERROR' + + @override + def __init__(self, build_ctx: CTX_HINT | None = None): + super().__init__(build_ctx) + self.returns = [] + + def add(self, _bytes: bytes | EnhancedSmlFrame | str): + if isinstance(_bytes, EnhancedSmlFrame): + self.returns.append(_bytes) + elif isinstance(_bytes, str): + assert _bytes in 'CRC_ERROR' + self.returns.append(_bytes) + elif isinstance(_bytes, bytes): + self.returns.append(None) + super().add(_bytes) + elif _bytes is None: + pass + else: + raise TypeError() + + def clear(self): + super().clear() + + def get_frame(self) -> EnhancedSmlFrame | None: + if not self.returns: + return super().get_frame() + + cmd = self.returns.pop(0) + if cmd is None: + return super().get_frame() + + if isinstance(cmd, EnhancedSmlFrame): + return cmd + + if cmd == 'CRC_ERROR': + raise CrcError(b'my_msg', 123456, 654321) + + raise ValueError() diff --git a/tests/device/test_data.py b/tests/sml_data.py similarity index 53% rename from tests/device/test_data.py rename to tests/sml_data.py index ee20fa9..f2524c0 100644 --- a/tests/device/test_data.py +++ b/tests/sml_data.py @@ -1,43 +1,86 @@ -import logging -from asyncio import Task from binascii import a2b_hex -from unittest.mock import Mock -from serial_asyncio import SerialTransport +import pytest +from smllib.reader import SmlStreamReader -from sml2mqtt.device import Device, SmlSerial +from sml2mqtt.const import EnhancedSmlFrame, SmlFrameValues -async def test_serial_data(device: Device, no_serial, caplog, sml_data_1: bytes, arg_analyze): - caplog.set_level(logging.DEBUG) +@pytest.fixture() +def sml_data_1(): + return a2b_hex( + b'1B1B1B1B01010101760501188E6162006200726500000101760101070000000000000B000000000000000000000101636877007' + b'60501188E626200620072650000070177010B000000000000000000000172620165002EC3F47A77078181C78203FF0101010104' + b'45425A0177070100000009FF010101010B000000000000000000000177070100010800FF6401018001621E52FB690000000A7AC' + b'1BC170177070100010801FF0101621E52FB690000000A74B1EA770177070100010802FF0101621E52FB6900000000060FD1A001' + b'77070100020800FF6401018001621E52FB69000000000D19E1C00177070100100700FF0101621B52FE55000089D901770701002' + b'40700FF0101621B52FE55000020220177070100380700FF0101621B52FE5500000A9201770701004C0700FF0101621B52FE5500' + b'005F2501010163810200760501188E636200620072650000020171016325FC000000001B1B1B1B1A0356F5' + ) - # we want to test incoming data from the serial port - device.serial = SmlSerial() - device.serial.device = device - device.serial.transport = Mock(SerialTransport) - device.serial._task = Mock(Task) - chunk_size = 100 - for i in range(0, len(sml_data_1), chunk_size): - device.serial.data_received(a2b_hex(sml_data_1[i: i + chunk_size])) +@pytest.fixture() +def sml_frame_1(stream_reader): + frame = EnhancedSmlFrame(a2b_hex( + b'760500531efa620062007263010176010105001bb4fe0b0a0149534b0005020de272620165001bb32e620163a71400760500531e' + b'fb620062007263070177010b0a0149534b0005020de2070100620affff72620165001bb32e757707010060320101010101010449' + b'534b0177070100600100ff010101010b0a0149534b0005020de20177070100010800ff65001c010401621e52ff650026bea90177' + b'070100020800ff0101621e52ff62000177070100100700ff0101621b52005301100101016350ba00760500531efc620062007263' + b'0201710163ba1900' + )) - msg = "\n".join(x.msg for x in filter(lambda x: x.name == 'sml.mqtt.pub', caplog.records)) + stream_reader.add(frame) + return frame - assert msg == \ - 'testing/00000000000000000000/0100010800ff: 450.09189911 (QOS: 0, retain: False)\n' \ - 'testing/00000000000000000000/0100010801ff: 449.07489911 (QOS: 0, retain: False)\n' \ - 'testing/00000000000000000000/0100010802ff: 1.017 (QOS: 0, retain: False)\n' \ - 'testing/00000000000000000000/0100020800ff: 2.198 (QOS: 0, retain: False)\n' \ - 'testing/00000000000000000000/0100100700ff: 352.89 (QOS: 0, retain: False)\n' \ - 'testing/00000000000000000000/0100240700ff: 82.26 (QOS: 0, retain: False)\n' \ - 'testing/00000000000000000000/0100380700ff: 27.06 (QOS: 0, retain: False)\n' \ - 'testing/00000000000000000000/01004c0700ff: 243.57 (QOS: 0, retain: False)\n' \ - 'testing/00000000000000000000/status: OK (QOS: 0, retain: False)\n' \ - 'testing/00000000000000000000/status: SHUTDOWN (QOS: 0, retain: False)' - msg = "\n".join(x.msg for x in filter(lambda x: x.name == 'sml.device_url', caplog.records)) +@pytest.fixture() +def sml_frame_2(stream_reader): + frame = EnhancedSmlFrame(a2b_hex( + b'7605065850a66200620072630101760107ffffffffffff05021d70370b0a014c475a0003403b4972620165021d7707016326de' + b'007605065850a762006200726307017707ffffffffffff0b0a014c475a0003403b49070100620affff72620165021d77077577' + b'0701006032010101010101044c475a0177070100600100ff010101010b0a014c475a0003403b490177070100010800ff65001c' + b'010472620165021d7707621e52ff690000000003152c450177070100020800ff0172620165021d7707621e52ff690000000000' + b'0000000177070100100700ff0101621b52005900000000000000fb010101637264007605065850a86200620072630201710163' + b'1c8c00' + )) - assert msg == ''' + stream_reader.add(frame) + return frame + + +@pytest.fixture() +def sml_data_1_analyze(sml_data_1): + r = SmlStreamReader() + r.add(sml_data_1) + frame = r.get_frame() # type: EnhancedSmlFrame | None + return '\n'.join(frame.get_analyze_str()) + + +@pytest.fixture() +def sml_frame_1_values(sml_frame_1): + values = sml_frame_1.get_obis() + return SmlFrameValues.create(0, values) + + +@pytest.fixture() +def sml_frame_1_analyze(sml_frame_1): + return '\n'.join(sml_frame_1.get_analyze_str()) + + +@pytest.fixture() +def sml_frame_2_values(sml_frame_2): + values = sml_frame_2.get_obis() + return SmlFrameValues.create(0, values) + + +@pytest.fixture() +def sml_frame_2_analyze(sml_frame_2): + return '\n'.join(sml_frame_2.get_analyze_str()) + + +@pytest.fixture() +def sml_data_1_analyze(): + return """ Received Frame -> b'760501188e6162006200726500000101760101070000000000000b00000000000000000000010163687700760501188e626200620072650000070177010b000000000000000000000172620165002ec3f47a77078181c78203ff010101010445425a0177070100000009ff010101010b000000000000000000000177070100010800ff6401018001621e52fb690000000a7ac1bc170177070100010801ff0101621e52fb690000000a74b1ea770177070100010802ff0101621e52fb6900000000060fd1a00177070100020800ff6401018001621e52fb69000000000d19e1c00177070100100700ff0101621b52fe55000089d90177070100240700ff0101621b52fe55000020220177070100380700ff0101621b52fe5500000a9201770701004c0700ff0101621b52fe5500005f2501010163810200760501188e636200620072650000020171016325fc00' @@ -89,7 +132,7 @@ async def test_serial_data(device: Device, no_serial, caplog, sml_data_1: bytes, scaler : -5 value : 45009189911 value_signature: None - -> 450091.89911Wh (Zählerstand Total) + -> 450091.89911Wh (Zählerstand Bezug Total) obis : 0100010801ff (1-0:1.8.1*255) status : None @@ -98,7 +141,7 @@ async def test_serial_data(device: Device, no_serial, caplog, sml_data_1: bytes, scaler : -5 value : 44907489911 value_signature: None - -> 449074.89911Wh (Zählerstand Tarif 1) + -> 449074.89911Wh (Zählerstand Bezug Tarif 1) obis : 0100010802ff (1-0:1.8.2*255) status : None @@ -107,7 +150,7 @@ async def test_serial_data(device: Device, no_serial, caplog, sml_data_1: bytes, scaler : -5 value : 101700000 value_signature: None - -> 1017.0Wh (Zählerstand Tarif 2) + -> 1017.0Wh (Zählerstand Bezug Tarif 2) obis : 0100020800ff (1-0:2.8.0*255) status : 65920 @@ -116,7 +159,7 @@ async def test_serial_data(device: Device, no_serial, caplog, sml_data_1: bytes, scaler : -5 value : 219800000 value_signature: None - -> 2198.0Wh (Wirkenergie Total) + -> 2198.0Wh (Zählerstand Einspeisung Total) obis : 0100100700ff (1-0:16.7.0*255) status : None @@ -134,7 +177,7 @@ async def test_serial_data(device: Device, no_serial, caplog, sml_data_1: bytes, scaler : -2 value : 8226 value_signature: None - -> 82.26W (Wirkleistung L1) + -> 82.26W (Summenwirkleistung L1) obis : 0100380700ff (1-0:56.7.0*255) status : None @@ -143,7 +186,7 @@ async def test_serial_data(device: Device, no_serial, caplog, sml_data_1: bytes, scaler : -2 value : 2706 value_signature: None - -> 27.06W (Wirkleistung L2) + -> 27.06W (Summenwirkleistung L2) obis : 01004c0700ff (1-0:76.7.0*255) status : None @@ -152,7 +195,7 @@ async def test_serial_data(device: Device, no_serial, caplog, sml_data_1: bytes, scaler : -2 value : 24357 value_signature: None - -> 243.57W (Wirkleistung L3) + -> 243.57W (Summenwirkleistung L3) list_signature : None act_gateway_time: None crc16 : 33026 @@ -163,71 +206,4 @@ async def test_serial_data(device: Device, no_serial, caplog, sml_data_1: bytes, message_body global_signature: None crc16 : 9724 - -Found obis id 0100000009ff in the sml frame -No configuration found for 00000000000000000000 -Creating default value handler for 0100010800ff -Creating default value handler for 0100010801ff -Creating default value handler for 0100010802ff -Creating default value handler for 0100020800ff -Creating default value handler for 0100100700ff -Creating default value handler for 0100240700ff -Creating default value handler for 0100380700ff -Creating default value handler for 01004c0700ff - -testing/00000000000000000000/0100010800ff (0100010800ff): - raw value: 450.09189911 - pub value: 450.09189911 - filters: - - - - - -testing/00000000000000000000/0100010801ff (0100010801ff): - raw value: 449.07489911 - pub value: 449.07489911 - filters: - - - - - -testing/00000000000000000000/0100010802ff (0100010802ff): - raw value: 1.017 - pub value: 1.017 - filters: - - - - - -testing/00000000000000000000/0100020800ff (0100020800ff): - raw value: 2.198 - pub value: 2.198 - filters: - - - - - -testing/00000000000000000000/0100100700ff (0100100700ff): - raw value: 352.89 - pub value: 352.89 - filters: - - - - - -testing/00000000000000000000/0100240700ff (0100240700ff): - raw value: 82.26 - pub value: 82.26 - filters: - - - - - -testing/00000000000000000000/0100380700ff (0100380700ff): - raw value: 27.06 - pub value: 27.06 - filters: - - - - - -testing/00000000000000000000/01004c0700ff (01004c0700ff): - raw value: 243.57 - pub value: 243.57 - filters: - - - - -''' +""" diff --git a/tests/device/__init__.py b/tests/sml_device/__init__.py similarity index 100% rename from tests/device/__init__.py rename to tests/sml_device/__init__.py diff --git a/tests/sml_device/frames/test_frame_1.py b/tests/sml_device/frames/test_frame_1.py new file mode 100644 index 0000000..4eca213 --- /dev/null +++ b/tests/sml_device/frames/test_frame_1.py @@ -0,0 +1,155 @@ +from binascii import a2b_hex + +import pytest + +from sml2mqtt import CONFIG +from sml2mqtt.config.device import SmlDeviceConfig +from sml2mqtt.sml_device import SmlDevice + + +@pytest.mark.ignore_log_errors() +async def test_frame_no_match_obis_id(no_mqtt, caplog, monkeypatch, sml_frame_1, arg_analyze, sml_frame_1_analyze): + device = SmlDevice('device_name') + device.frame_handler = device.analyze_frame + + monkeypatch.setattr(CONFIG.general, 'device_id_obis', ['0100000009ff', '01006001ffff']) + + device.on_source_data(None) + + msg = "\n".join(x.msg for x in caplog.records) + + assert msg.removeprefix(sml_frame_1_analyze) == ''' +Found none of the following obis ids in the sml frame: 0100000009ff, 01006001ffff +Received Frame + -> b'760500531efa620062007263010176010105001bb4fe0b0a0149534b0005020de272620165001bb32e620163a71400760500531efb620062007263070177010b0a0149534b0005020de2070100620affff72620165001bb32e757707010060320101010101010449534b0177070100600100ff010101010b0a0149534b0005020de20177070100010800ff65001c010401621e52ff650026bea90177070100020800ff0101621e52ff62000177070100100700ff0101621b52005301100101016350ba00760500531efc6200620072630201710163ba1900' +Exception : "" +ERROR +device_name/status: ERROR (QOS: 0, retain: False)''' + + +@pytest.mark.ignore_log_warnings() +async def test_frame_no_config(no_mqtt, caplog, monkeypatch, sml_frame_1, arg_analyze, sml_frame_1_analyze): + device = SmlDevice('device_name') + device.frame_handler = device.analyze_frame + + monkeypatch.setattr(CONFIG.general, 'device_id_obis', ['0100600100ff']) + + device.on_source_data(None) + + msg = "\n".join(x.msg for x in caplog.records) + + assert msg.removeprefix(sml_frame_1_analyze) == ''' +Found obis id 0100600100ff in the sml frame +No device found for 0a0149534b0005020de2 +No filters found for 010060320101, creating default filters +No filters found for 0100600100ff, creating default filters +No filters found for 0100010800ff, creating default filters +No filters found for 0100020800ff, creating default filters +No filters found for 0100100700ff, creating default filters +Skipped: 0100600100ff + + + obis : 010060320101 + topic: 0a0149534b0005020de2/010060320101 + operations: + - On Change Filter + - Refresh Action: 2 minutes + + + obis : 0100600100ff + topic: 0a0149534b0005020de2/0100600100ff + operations: + - On Change Filter + - Refresh Action: 2 minutes + + + obis : 0100010800ff + topic: 0a0149534b0005020de2/0100010800ff + operations: + - Zero Meter Filter + - Factor: 0.001 + - On Change Filter + - Refresh Action: 2 minutes + + + obis : 0100020800ff + topic: 0a0149534b0005020de2/0100020800ff + operations: + - Zero Meter Filter + - Factor: 0.001 + - On Change Filter + - Refresh Action: 2 minutes + + + obis : 0100100700ff + topic: 0a0149534b0005020de2/0100100700ff + operations: + - On Change Filter + - Refresh Action: 2 minutes + +0a0149534b0005020de2/010060320101: ISK (QOS: 0, retain: False) +0a0149534b0005020de2/0100600100ff: 0a0149534b0005020de2 (QOS: 0, retain: False) +0a0149534b0005020de2/0100010800ff: 253.91770000000002 (QOS: 0, retain: False) +0a0149534b0005020de2/0100100700ff: 272 (QOS: 0, retain: False) +OK +0a0149534b0005020de2/status: OK (QOS: 0, retain: False) + +SHUTDOWN +0a0149534b0005020de2/status: SHUTDOWN (QOS: 0, retain: False)''' + + +async def test_frame_with_config(no_mqtt, caplog, monkeypatch, sml_frame_1, arg_analyze, sml_frame_1_analyze): + device = SmlDevice('device_name') + device.frame_handler = device.analyze_frame + + monkeypatch.setattr(CONFIG.general, 'device_id_obis', ['0100600100ff']) + monkeypatch.setitem(CONFIG.devices, '0a0149534b0005020de2', SmlDeviceConfig( + skip=['010060320101'] + )) + + device.on_source_data(None) + + msg = "\n".join(x.msg for x in caplog.records) + + print(msg) + + assert msg.removeprefix(sml_frame_1_analyze) == ''' +Found obis id 0100600100ff in the sml frame +Device found for 0a0149534b0005020de2 +No filters found for 0100010800ff, creating default filters +No filters found for 0100020800ff, creating default filters +No filters found for 0100100700ff, creating default filters +Skipped: 0100600100ff, 010060320101 + + + obis : 0100010800ff + topic: 0a0149534b0005020de2/0100010800ff + operations: + - Zero Meter Filter + - Factor: 0.001 + - On Change Filter + - Refresh Action: 2 minutes + + + obis : 0100020800ff + topic: 0a0149534b0005020de2/0100020800ff + operations: + - Zero Meter Filter + - Factor: 0.001 + - On Change Filter + - Refresh Action: 2 minutes + + + obis : 0100100700ff + topic: 0a0149534b0005020de2/0100100700ff + operations: + - On Change Filter + - Refresh Action: 2 minutes + +0a0149534b0005020de2/0100010800ff: 253.91770000000002 (QOS: 0, retain: False) +0a0149534b0005020de2/0100100700ff: 272 (QOS: 0, retain: False) +OK +0a0149534b0005020de2/status: OK (QOS: 0, retain: False) + +SHUTDOWN +0a0149534b0005020de2/status: SHUTDOWN (QOS: 0, retain: False)''' diff --git a/tests/sml_device/frames/test_frame_2.py b/tests/sml_device/frames/test_frame_2.py new file mode 100644 index 0000000..734b66c --- /dev/null +++ b/tests/sml_device/frames/test_frame_2.py @@ -0,0 +1,27 @@ +from binascii import a2b_hex + +import pytest + +from sml2mqtt import CONFIG +from sml2mqtt.sml_device import SmlDevice + + +@pytest.mark.ignore_log_errors() +async def test_frame_no_match_obis_id(no_mqtt, caplog, monkeypatch, sml_frame_2, arg_analyze, sml_frame_2_analyze): + device = SmlDevice('device_name') + device.frame_handler = device.analyze_frame + + monkeypatch.setattr(CONFIG.general, 'device_id_obis', ['0100000009ff', '01006001ffff']) + + device.on_source_data(None) + + msg = "\n".join(x.msg for x in caplog.records) + + assert msg.removeprefix(sml_frame_2_analyze) == ''' +get_obis failed - try parsing frame +Found none of the following obis ids in the sml frame: 0100000009ff, 01006001ffff +Received Frame + -> b'7605065850a66200620072630101760107ffffffffffff05021d70370b0a014c475a0003403b4972620165021d7707016326de007605065850a762006200726307017707ffffffffffff0b0a014c475a0003403b49070100620affff72620165021d770775770701006032010101010101044c475a0177070100600100ff010101010b0a014c475a0003403b490177070100010800ff65001c010472620165021d7707621e52ff690000000003152c450177070100020800ff0172620165021d7707621e52ff6900000000000000000177070100100700ff0101621b52005900000000000000fb010101637264007605065850a862006200726302017101631c8c00' +Exception : "" +ERROR +device_name/status: ERROR (QOS: 0, retain: False)''' diff --git a/tests/sml_device/test_device.py b/tests/sml_device/test_device.py new file mode 100644 index 0000000..b52b259 --- /dev/null +++ b/tests/sml_device/test_device.py @@ -0,0 +1,121 @@ +from binascii import a2b_hex + +import pytest + +from sml2mqtt.sml_device import SmlDevice + + +@pytest.mark.ignore_log_warnings() +async def test_device_analyze(no_mqtt, caplog, sml_data_1, arg_analyze, sml_data_1_analyze): + device = SmlDevice('device_name') + device.frame_handler = device.analyze_frame + + # feed data in chunks + chunk_size = 100 + for i in range(0, len(sml_data_1), chunk_size): + device.on_source_data(sml_data_1[i: i + chunk_size]) + + # This is what will be reported + msg = "\n".join(x.msg for x in filter(lambda x: x.name == 'sml.mqtt.pub', caplog.records)) + + assert '\n' + msg == """ +00000000000000000000/0100000009ff: 00000000000000000000 (QOS: 0, retain: False) +00000000000000000000/0100010800ff: 450.09189911 (QOS: 0, retain: False) +00000000000000000000/0100010801ff: 449.07489911 (QOS: 0, retain: False) +00000000000000000000/0100010802ff: 1.0170000000000001 (QOS: 0, retain: False) +00000000000000000000/0100020800ff: 2.198 (QOS: 0, retain: False) +00000000000000000000/0100100700ff: 352.89 (QOS: 0, retain: False) +00000000000000000000/0100240700ff: 82.26 (QOS: 0, retain: False) +00000000000000000000/0100380700ff: 27.06 (QOS: 0, retain: False) +00000000000000000000/01004c0700ff: 243.57 (QOS: 0, retain: False) +00000000000000000000/status: OK (QOS: 0, retain: False) +00000000000000000000/status: SHUTDOWN (QOS: 0, retain: False)""" + + msg = "\n".join(x.msg for x in filter(lambda x: x.name == 'sml.device_name', caplog.records)) + + assert msg.removeprefix(sml_data_1_analyze) == ''' +Found obis id 0100000009ff in the sml frame +No device found for 00000000000000000000 +No filters found for 0100000009ff, creating default filters +No filters found for 0100010800ff, creating default filters +No filters found for 0100010801ff, creating default filters +No filters found for 0100010802ff, creating default filters +No filters found for 0100020800ff, creating default filters +No filters found for 0100100700ff, creating default filters +No filters found for 0100240700ff, creating default filters +No filters found for 0100380700ff, creating default filters +No filters found for 01004c0700ff, creating default filters +Skipped: 0100000009ff, 0100600100ff + + + obis : 0100000009ff + topic: 00000000000000000000/0100000009ff + operations: + - On Change Filter + - Refresh Action: 2 minutes + + + obis : 0100010800ff + topic: 00000000000000000000/0100010800ff + operations: + - Zero Meter Filter + - Factor: 0.001 + - On Change Filter + - Refresh Action: 2 minutes + + + obis : 0100010801ff + topic: 00000000000000000000/0100010801ff + operations: + - Zero Meter Filter + - Factor: 0.001 + - On Change Filter + - Refresh Action: 2 minutes + + + obis : 0100010802ff + topic: 00000000000000000000/0100010802ff + operations: + - Zero Meter Filter + - Factor: 0.001 + - On Change Filter + - Refresh Action: 2 minutes + + + obis : 0100020800ff + topic: 00000000000000000000/0100020800ff + operations: + - Zero Meter Filter + - Factor: 0.001 + - On Change Filter + - Refresh Action: 2 minutes + + + obis : 0100100700ff + topic: 00000000000000000000/0100100700ff + operations: + - On Change Filter + - Refresh Action: 2 minutes + + + obis : 0100240700ff + topic: 00000000000000000000/0100240700ff + operations: + - On Change Filter + - Refresh Action: 2 minutes + + + obis : 0100380700ff + topic: 00000000000000000000/0100380700ff + operations: + - On Change Filter + - Refresh Action: 2 minutes + + + obis : 01004c0700ff + topic: 00000000000000000000/01004c0700ff + operations: + - On Change Filter + - Refresh Action: 2 minutes + +''' diff --git a/tests/sml_device/test_setup_device.py b/tests/sml_device/test_setup_device.py new file mode 100644 index 0000000..afbeb08 --- /dev/null +++ b/tests/sml_device/test_setup_device.py @@ -0,0 +1,36 @@ +import logging + +import pytest + +from sml2mqtt.config.config import GeneralSettings +from sml2mqtt.config.device import SmlDeviceConfig, SmlValueConfig +from sml2mqtt.config.operations import OnChangeFilter +from sml2mqtt.sml_device import SmlDevice +from sml2mqtt.sml_device.setup_device import setup_device +from sml2mqtt.sml_value.operations import OnChangeFilterOperation + + +@pytest.mark.ignore_log_warnings() +def test_warnings(no_mqtt, caplog, sml_frame_1_values): + device = SmlDevice('test_device') + device.mqtt_device.cfg.topic_full = 'test_device/device_id' + + device_cfg = SmlDeviceConfig( + skip={'0100000009ff'}, + values=[ + SmlValueConfig(obis='0100000009ff', operations=[{'type': 'change filter'}]) + ] + ) + general_cfg = GeneralSettings() + + setup_device(device, sml_frame_1_values, device_cfg, general_cfg) + + # This is what will be reported + msg = "\n".join( + x.msg for x in filter(lambda x: x.name == 'sml.test_device' and x.levelno == logging.WARNING, caplog.records) + ) + + assert '\n' + msg + '\n' == """ +Config for 0100000009ff found but 0100000009ff is also marked to be skipped +Config for 0100000009ff found but 0100000009ff was not reported by the frame +""" diff --git a/tests/sml_device/test_watchdog.py b/tests/sml_device/test_watchdog.py new file mode 100644 index 0000000..73ce070 --- /dev/null +++ b/tests/sml_device/test_watchdog.py @@ -0,0 +1,78 @@ +from __future__ import annotations + +import asyncio +from unittest.mock import Mock + +from sml2mqtt.sml_device import DeviceStatus, SmlDevice +from sml2mqtt.sml_device.watchdog import Watchdog + + +def get_watchdog() -> tuple[Mock, Watchdog]: + p = Mock() + p.name = 'test' + m = p.on_timeout + m.assert_not_called() + + w = Watchdog(p).set_timeout(0.1) + return m, w + + +async def test_watchdog_expire(): + + m, w = get_watchdog() + w.start() + + await asyncio.sleep(0.15) + m.assert_called_once() + w.feed() + await asyncio.sleep(0.15) + assert m.call_count == 2 + + await w.cancel_and_wait() + + # Assert that the task is properly canceled + await asyncio.sleep(0.05) + + +async def test_watchdog_no_expire(): + + m, w = get_watchdog() + w.start() + + for _ in range(4): + w.feed() + await asyncio.sleep(0.06) + + m.assert_not_called() + + await w.cancel_and_wait() + + # Assert that the task is properly canceled + await asyncio.sleep(0.05) + + +async def test_watchdog_setup_and_feed(sml_data_1): + + obj = SmlDevice('test') + obj.frame_handler = obj.process_frame + obj.sml_values.set_skipped( + '0100000009ff', '0100010800ff', '0100010801ff', '0100010802ff', '0100020800ff', + '0100100700ff', '0100240700ff', '0100380700ff', '01004c0700ff' + ) + obj.watchdog.set_timeout(0.2) + + await obj.start() + assert obj.status == DeviceStatus.STARTUP + + await asyncio.sleep(0.3) + assert obj.status == DeviceStatus.MSG_TIMEOUT + + for _ in range(5): + await asyncio.sleep(0.15) + obj.on_source_data(sml_data_1) + assert obj.status == DeviceStatus.OK + + await asyncio.sleep(0.3) + assert obj.status == DeviceStatus.MSG_TIMEOUT + + await obj.cancel_and_wait() diff --git a/tests/values/__init__.py b/tests/sml_values/__init__.py similarity index 100% rename from tests/values/__init__.py rename to tests/sml_values/__init__.py diff --git a/tests/values/filters/__init__.py b/tests/sml_values/test_operations/__init__.py similarity index 100% rename from tests/values/filters/__init__.py rename to tests/sml_values/test_operations/__init__.py diff --git a/tests/sml_values/test_operations/helper.py b/tests/sml_values/test_operations/helper.py new file mode 100644 index 0000000..d794ef2 --- /dev/null +++ b/tests/sml_values/test_operations/helper.py @@ -0,0 +1,52 @@ +import re +from collections.abc import Iterable +from itertools import zip_longest + +from sml2mqtt.sml_value.base import ValueOperationBase + + +RE_ID = re.compile(r' at 0x[0-f]{6,}>') + + +def check_operation_repr(obj: ValueOperationBase, *values): + repr_str = RE_ID.sub('', repr(obj)) + + class_name = obj.__class__.__name__ + for suffix in ('Operation', ): + class_name = class_name.removesuffix(suffix) + + values_str = ' '.join(values) + if values_str: + values_str = ': ' + values_str + + target = f'<{class_name:s}{values_str:s}' + + assert target == repr_str, f'\n{target}\n{repr_str}' + + +def check_description(obj: ValueOperationBase, value: str | Iterable[str]): + desc = list(obj.describe()) + desc_text = '\n'.join(desc) + if 'filter' in desc_text.lower(): + assert 'Filter' in desc_text, desc_text + + # Object names should be title case + for line in desc: + if ' - ' in line or line.startswith('- '): + if '- 2001' in line or '- now' in line: + continue + + if 'On Change Filter' not in line and 'Zero Meter Filter' not in line and 'Negative On Status' not in line: + assert ':' in line, line + if ':' in line: + line = line[:line.index(':')] + line = line.removesuffix('0100010800ff') + assert line == line.title(), f'\n{line}\n{line.title()}' + + value = [value] if isinstance(value, str) else list(value) + diffs = [ + ''.join('^' if a != b else ' ' for a, b in zip_longest(entry_d, entry_v)) + for entry_d, entry_v in zip(desc, value) + ] + + assert desc == value, f'\n{desc}\n{value}\n{diffs}' diff --git a/tests/sml_values/test_operations/test_actions.py b/tests/sml_values/test_operations/test_actions.py new file mode 100644 index 0000000..d2dce04 --- /dev/null +++ b/tests/sml_values/test_operations/test_actions.py @@ -0,0 +1,59 @@ +from tests.sml_values.test_operations.helper import check_description, check_operation_repr + +from sml2mqtt.sml_value.operations import HeartbeatActionOperation, RefreshActionOperation +from sml2mqtt.sml_value.operations._helper import format_period + + +def test_format_period(): + assert format_period(30.2) == '30.2 seconds' + assert format_period(30) == '30 seconds' + assert format_period(60) == '1 minute' + assert format_period(61) == '1 minute 1 second' + assert format_period(121) == '2 minutes 1 second' + assert format_period(3661) == '1 hour 1 minute 1 second' + assert format_period(3722) == '1 hour 2 minutes 2 seconds' + + +def test_refresh_action(monotonic): + f = RefreshActionOperation(30) + check_operation_repr(f, '30s') + check_description(f, '- Refresh Action: 30 seconds') + + assert f.process_value(1, None) == 1 + assert f.process_value(None, None) is None + + monotonic.add(5) + assert f.process_value(2, None) == 2 + assert f.process_value(None, None) is None + + monotonic.add(29.99) + assert f.process_value(None, None) is None + + monotonic.add(0.02) + assert f.process_value(None, None) == 2 + + +def test_heartbeat_action(monotonic): + f = HeartbeatActionOperation(30) + check_operation_repr(f, '30s') + + assert f.process_value(1, None) == 1 + + monotonic.add(15) + assert f.process_value(2, None) is None + + monotonic.add(14.99) + assert f.process_value(3, None) is None + + monotonic.add(0.01) + assert f.process_value(None, None) == 3 + assert f.process_value(2, None) is None + + monotonic.add(30.01) + assert f.process_value(5, None) == 5 + assert f.process_value(5, None) is None + + check_description( + HeartbeatActionOperation(30), + '- Heartbeat Action: 30 seconds' + ) diff --git a/tests/sml_values/test_operations/test_date_time.py b/tests/sml_values/test_operations/test_date_time.py new file mode 100644 index 0000000..8cbc711 --- /dev/null +++ b/tests/sml_values/test_operations/test_date_time.py @@ -0,0 +1,368 @@ +from datetime import datetime, time + +import pytest +from tests.sml_values.test_operations.helper import check_description + +from sml2mqtt.const import date_time_finder as date_time_finder_module +from sml2mqtt.sml_value.operations import DateTimeFinder, MaxValueOperation, MinValueOperation, VirtualMeterOperation +from sml2mqtt.sml_value.operations import date_time as virtual_meter_module + + +class PatchedNow: + def __init__(self): + self.ret = None + + def set(self, dt: datetime): + self.ret = dt + return self.ret + + def __call__(self): + assert self.ret + return self.ret + + +class DateTimeFactory: + def __init__(self, year: int | None = 2001, month: int | None = 1, day: int | None = None, + hour: int | None = None, minute: int | None = None, second: int | None = 0, + microsecond: int | None = 0): + self.kwargs = { + 'year': year, 'month': month, 'day': day, + 'hour': hour, 'minute': minute, 'second': second, 'microsecond': microsecond + } + + def create(self, *args, **kwargs): + + call = {} + pos = 0 + for name, value in self.kwargs.items(): + value = kwargs.get(name, value) + if value is None and pos < len(args): + value = args[pos] + pos += 1 + + call[name] = value + + assert len(args) == pos + + return datetime(**call) + + +@pytest.fixture() +def now(monkeypatch): + p = PatchedNow() + monkeypatch.setattr(virtual_meter_module, 'get_now', p) + monkeypatch.setattr(date_time_finder_module, 'get_now', p) + return p + + +def test_finder_1(now): + f = DateTimeFinder() + f.add_time(time(2)) + + dt_set = DateTimeFactory(hour=1, minute=30) + dt_next = DateTimeFactory(hour=2, minute=0) + + for i in range(1, 31): + now.set(dt_set.create(i)) + assert f.calc_next() == dt_next.create(i) + + +def test_finder_dow(now): + f = DateTimeFinder() + f.add_time(time(2)) + f.add_dow(1) + + dt_set = DateTimeFactory(hour=1, minute=30) + dt_next = DateTimeFactory(hour=2, minute=0) + + for i in range(1, 30, 7): + now.set(dt_set.create(i)) + assert f.calc_next() == dt_next.create(i) + + +def test_finder_day(now): + f = DateTimeFinder() + f.add_time(time(2)) + f.add_day(15) + f.add_day(31) + + dt_set = DateTimeFactory(hour=1, minute=30) + dt_next = DateTimeFactory(hour=2, minute=0) + + now.set(dt_set.create(15)) + assert f.calc_next() == dt_next.create(15) + + now.set(dt_set.create(31)) + assert f.calc_next() == dt_next.create(31) + + now.set(dt_set.create(15, month=2)) + assert f.calc_next() == dt_next.create(15, month=2) + + now.set(dt_set.create(15, month=3)) + assert f.calc_next() == dt_next.create(15, month=3) + + +def test_virtual_meter_start_now(now): + f = DateTimeFinder() + f.add_time(time(2)) + + dt = DateTimeFactory(hour=1, minute=30) + now.set(dt.create(1, microsecond=1)) + + o = VirtualMeterOperation(f, start_now=True) + + assert o.process_value(None, None) is None + assert o.process_value(33, None) == 0 + assert o.process_value(34, None) == 1 + + +def test_virtual_meter_start_normal(now): + f = DateTimeFinder() + f.add_time(time(2)) + + dt = DateTimeFactory(hour=1, minute=30) + now.set(dt.create(1)) + + o = VirtualMeterOperation(f, start_now=False) + now.set(dt.create(1)) + + assert o.process_value(None, None) is None + assert o.process_value(33, None) is None + assert o.process_value(34, None) is None + + now.set(dt.create(1, hour=2, minute=0, second=1)) + assert o.process_value(35, None) == 1 + + +def test_virtual_meter_description(now): + f = DateTimeFinder() + f.add_time(time(2)) + + dt = DateTimeFactory(hour=1, minute=30) + now.set(dt.create(1)) + + o = VirtualMeterOperation(f, start_now=True) + now.set(dt.create(1)) + + assert o.process_value(1, None) == 0 + + check_description( + o, [ + '- Virtual Meter:', + ' Offset: 1', + ' Next resets:', + ' - 2001-01-01 02:00:00', + ' - 2001-01-02 02:00:00', + ' - 2001-01-03 02:00:00', + ] + ) + + +def test_virtual_meter_start_now_no_times(now): + f = DateTimeFinder() + f.add_time(time(2)) + + dt = DateTimeFactory(hour=1, minute=30) + now.set(dt.create(1, microsecond=1)) + + o = VirtualMeterOperation(f, start_now=True) + + assert o.process_value(None, None) is None + assert o.process_value(33, None) == 0 + assert o.process_value(34, None) == 1 + + +def test_virtual_meter_start_normal_no_times(now): + f = DateTimeFinder() + + dt = DateTimeFactory(hour=1, minute=30) + now.set(dt.create(1)) + + o = VirtualMeterOperation(f, start_now=False) + now.set(dt.create(1)) + + assert o.process_value(None, None) is None + assert o.process_value(33, None) == 0 + assert o.process_value(34, None) == 1 + + +def test_virtual_meter_description_no_times(now): + f = DateTimeFinder() + + dt = DateTimeFactory(hour=1, minute=30) + now.set(dt.create(1)) + + o = VirtualMeterOperation(f, start_now=True) + now.set(dt.create(1)) + + assert o.process_value(1, None) == 0 + + check_description( + o, [ + '- Virtual Meter:', + ' Offset: 1', + ' No resets', + ] + ) + + +def test_max_start_now(now): + f = DateTimeFinder() + f.add_time(time(2)) + + dt = DateTimeFactory(hour=1, minute=30) + now.set(dt.create(1, microsecond=1)) + + o = MaxValueOperation(f, start_now=True) + + assert o.process_value(None, None) is None + assert o.process_value(50, None) == 50 + assert o.process_value(33, None) is None + assert o.process_value(50, None) is None + assert o.process_value(51, None) == 51 + assert o.process_value(50, None) is None + + +def test_max_start_normal(now): + f = DateTimeFinder() + f.add_time(time(2)) + + dt = DateTimeFactory(day=1, hour=1, minute=0) + now.set(dt.create(microsecond=1)) + + o = MaxValueOperation(f, start_now=False) + + assert o.process_value(None, None) is None + assert o.process_value(9999, None) is None + + now.set(dt.create(hour=2)) + + assert o.process_value(None, None) is None + assert o.process_value(50, None) == 50 + assert o.process_value(33, None) is None + assert o.process_value(50, None) is None + assert o.process_value(51, None) == 51 + assert o.process_value(50, None) is None + + +def test_max_description(now): + f = DateTimeFinder() + f.add_time(time(2)) + + dt = DateTimeFactory(day=1, hour=1, minute=0) + now.set(dt.create(microsecond=1)) + + o = MaxValueOperation(f, start_now=True) + assert o.process_value(1, None) == 1 + + check_description( + o, [ + '- Max Value:', + ' max: 1', + ' Next resets:', + ' - 2001-01-01 02:00:00', + ' - 2001-01-02 02:00:00', + ' - 2001-01-03 02:00:00', + ] + ) + + o = MaxValueOperation(f, start_now=False) + assert o.process_value(1, None) is None + + check_description( + o, [ + '- Max Value:', + ' max: None', + ' Next resets:', + ' - 2001-01-01 02:00:00', + ' - 2001-01-02 02:00:00', + ' - 2001-01-03 02:00:00', + ] + ) + + now.set(datetime(2001, 1, 1, 2, 0, 1)) + check_description( + o, [ + '- Max Value:', + ' max: None', + ' Next resets:', + ' - now', + ' - 2001-01-02 02:00:00', + ' - 2001-01-03 02:00:00', + ] + ) + + +def test_min_start_now(now): + f = DateTimeFinder() + f.add_time(time(2)) + + dt = DateTimeFactory(hour=1, minute=30) + now.set(dt.create(1, microsecond=1)) + + o = MinValueOperation(f, start_now=True) + + assert o.process_value(None, None) is None + assert o.process_value(50, None) == 50 + assert o.process_value(55, None) is None + assert o.process_value(50, None) is None + assert o.process_value(49, None) == 49 + assert o.process_value(49, None) is None + + +def test_min_start_normal(now): + f = DateTimeFinder() + f.add_time(time(2)) + + dt = DateTimeFactory(day=1, hour=1, minute=0) + now.set(dt.create(microsecond=1)) + + o = MinValueOperation(f, start_now=False) + + assert o.process_value(None, None) is None + assert o.process_value(-9999, None) is None + + now.set(dt.create(hour=2)) + + assert o.process_value(None, None) is None + assert o.process_value(50, None) == 50 + assert o.process_value(55, None) is None + assert o.process_value(50, None) is None + assert o.process_value(49, None) == 49 + assert o.process_value(49, None) is None + + +def test_min_description(now): + f = DateTimeFinder() + f.add_time(time(2)) + + dt = DateTimeFactory(day=1, hour=1, minute=0) + now.set(dt.create(microsecond=1)) + + o = MinValueOperation(f, start_now=True) + assert o.process_value(1, None) == 1 + + check_description( + o, [ + '- Min Value:', + ' min: 1', + ' Next resets:', + ' - 2001-01-01 02:00:00', + ' - 2001-01-02 02:00:00', + ' - 2001-01-03 02:00:00', + ] + ) + + o = MinValueOperation(f, start_now=False) + assert o.process_value(1, None) is None + + check_description( + o, [ + '- Min Value:', + ' min: None', + ' Next resets:', + ' - 2001-01-01 02:00:00', + ' - 2001-01-02 02:00:00', + ' - 2001-01-03 02:00:00', + ] + ) diff --git a/tests/sml_values/test_operations/test_filter.py b/tests/sml_values/test_operations/test_filter.py new file mode 100644 index 0000000..17e17d2 --- /dev/null +++ b/tests/sml_values/test_operations/test_filter.py @@ -0,0 +1,177 @@ +from tests.sml_values.test_operations.helper import check_description, check_operation_repr + +from sml2mqtt.sml_value.operations import ( + DeltaFilterOperation, + OnChangeFilterOperation, + RangeFilterOperation, + SkipZeroMeterOperation, + ThrottleFilterOperation, +) + + +def test_skip(): + f = SkipZeroMeterOperation() + check_operation_repr(f) + + assert f.process_value(0, None) is None + assert f.process_value(0.01, None) is None + assert f.process_value(1, None) == 1 + assert f.process_value(1.1, None) == 1.1 + + check_description( + SkipZeroMeterOperation(), + '- Zero Meter Filter' + ) + + +def test_delta(): + f = DeltaFilterOperation(min_value=5) + check_operation_repr(f, 'min=5 min_percent=None') + check_description(f, ['- Delta Filter:', ' Min : 5']) + + assert f.process_value(10, None) == 10 + assert f.process_value(14.999, None) is None + assert f.process_value(5.001, None) is None + + assert f.process_value(15, None) == 15 + assert f.process_value(10.0001, None) is None + assert f.process_value(19.9999, None) is None + assert f.process_value(10, None) == 10 + + f = DeltaFilterOperation(min_percent=5) + check_operation_repr(f, 'min=None min_percent=5') + check_description(f, ['- Delta Filter:', ' Min %: 5']) + + assert f.process_value(0, None) == 0 + assert f.process_value(0.049, None) == 0.049 + + assert f.process_value(100, None) == 100 + assert f.process_value(104.999, None) is None + assert f.process_value(95.001, None) is None + + assert f.process_value(105, None) == 105 + assert f.process_value(109.999, None) is None + assert f.process_value(99.750001, None) is None + assert f.process_value(99.75, None) == 99.75 + + f = DeltaFilterOperation(min_value=5, min_percent=10) + check_operation_repr(f, 'min=5 min_percent=10') + check_description(f, ['- Delta Filter:', ' Min : 5', ' Min %: 10']) + + assert f.process_value(100, None) == 100 + assert f.process_value(109.99, None) is None + assert f.process_value(110, None) == 110 + + assert f.process_value(10, None) == 10 + assert f.process_value(14.99, None) is None + assert f.process_value(15, None) == 15 + + +def test_on_change(): + f = OnChangeFilterOperation() + check_operation_repr(f) + + assert f.process_value(10, None) == 10 + assert f.process_value(10, None) is None + assert f.process_value(11, None) == 11 + assert f.process_value(0, None) == 0 + + check_description( + OnChangeFilterOperation(), + '- On Change Filter' + ) + + +def test_range(): + + # --------------------------------------------------------------------------------------------- + # Min + o = RangeFilterOperation(1, None, True) + check_operation_repr(o, 'min=1 max=None limit_values=True') + + assert o.process_value(None, None) is None + assert o.process_value(5, None) == 5 + assert o.process_value(1, None) == 1 + assert o.process_value(0.999, None) == 1 + + o = RangeFilterOperation(1, None, False) + check_operation_repr(o, 'min=1 max=None limit_values=False') + + assert o.process_value(None, None) is None + assert o.process_value(1, None) == 1 + assert o.process_value(0.999, None) is None + + # --------------------------------------------------------------------------------------------- + # Max + o = RangeFilterOperation(None, 5, True) + check_operation_repr(o, 'min=None max=5 limit_values=True') + + assert o.process_value(None, None) is None + assert o.process_value(5, None) == 5 + assert o.process_value(4.99, None) == 4.99 + assert o.process_value(5.01, None) == 5 + + o = RangeFilterOperation(None, 5, False) + check_operation_repr(o, 'min=None max=5 limit_values=False') + + assert o.process_value(None, None) is None + assert o.process_value(5, None) == 5 + assert o.process_value(4.99, None) == 4.99 + assert o.process_value(5.01, None) is None + + # --------------------------------------------------------------------------------------------- + # Min Max + o = RangeFilterOperation(0, 5, True) + check_operation_repr(o, 'min=0 max=5 limit_values=True') + + assert o.process_value(None, None) is None + assert o.process_value(-0.001, None) == 0 + assert o.process_value(0.001, None) == 0.001 + assert o.process_value(4.999, None) == 4.999 + assert o.process_value(5.001, None) == 5 + + o = RangeFilterOperation(0, 5, False) + check_operation_repr(o, 'min=0 max=5 limit_values=False') + + assert o.process_value(None, None) is None + assert o.process_value(-0.001, None) is None + assert o.process_value(0, None) == 0 + assert o.process_value(5, None) == 5 + assert o.process_value(5.001, None) is None + + # LimitValueFilter + check_description( + RangeFilterOperation(1, None, False), + ['- Range Filter:', ' min: 1', ' limit to min/max: False'] + ) + check_description( + RangeFilterOperation(None, 7, False), + ['- Range Filter:', ' max: 7', ' limit to min/max: False'] + ) + check_description( + RangeFilterOperation(1, 7, True), [ + '- Range Filter:', + ' min: 1', + ' max: 7', + ' limit to min/max: True' + ] + ) + + +def test_throttle_filter(monotonic): + f = ThrottleFilterOperation(30) + check_operation_repr(f, '30s') + check_description(f, '- Throttle Filter: 30 seconds') + + assert f.process_value(None, None) is None + assert f.process_value(1, None) == 1 + + monotonic.set(29.99) + assert f.process_value(1, None) is None + monotonic.set(30) + assert f.process_value(1, None) == 1 + + monotonic.set(59.99) + assert f.process_value(1, None) is None + monotonic.set(60) + assert f.process_value(1, None) == 1 diff --git a/tests/sml_values/test_operations/test_math.py b/tests/sml_values/test_operations/test_math.py new file mode 100644 index 0000000..8210f10 --- /dev/null +++ b/tests/sml_values/test_operations/test_math.py @@ -0,0 +1,80 @@ +from tests.sml_values.test_operations.helper import check_description, check_operation_repr + +from sml2mqtt.sml_value.operations import ( + FactorOperation, + OffsetOperation, + RangeFilterOperation, + RoundOperation, +) + + +def test_factor(): + o = FactorOperation(5) + check_operation_repr(o, '5') + + assert o.process_value(None, None) is None + assert o.process_value(5, None) == 25 + assert o.process_value(1.25, None) == 6.25 + assert o.process_value(-3, None) == -15 + + +def test_offset(): + o = OffsetOperation(-5) + check_operation_repr(o, '-5') + + assert o.process_value(None, None) is None + assert o.process_value(5, None) == 0 + assert o.process_value(1.25, None) == -3.75 + assert o.process_value(-3, None) == -8 + + +def test_round(): + o = RoundOperation(0) + check_operation_repr(o, '0') + + assert o.process_value(None, None) is None + assert o.process_value(5, None) == 5 + assert o.process_value(1.25, None) == 1 + assert o.process_value(-3, None) == -3 + assert o.process_value(-3.65, None) == -4 + + o = RoundOperation(1) + check_operation_repr(o, '1') + + assert o.process_value(None, None) is None + assert o.process_value(5, None) == 5 + assert o.process_value(1.25, None) == 1.2 + assert o.process_value(-3, None) == -3 + assert o.process_value(-3.65, None) == -3.6 + + +def test_description(): + check_description( + FactorOperation(-5), + '- Factor: -5' + ) + + check_description( + FactorOperation(3.14), + '- Factor: 3.14' + ) + + check_description( + OffsetOperation(-5), + '- Offset: -5' + ) + + check_description( + OffsetOperation(3.14), + '- Offset: 3.14' + ) + + check_description( + RoundOperation(0), + '- Round: integer' + ) + + check_description( + RoundOperation(1), + '- Round: 1' + ) diff --git a/tests/sml_values/test_operations/test_operations.py b/tests/sml_values/test_operations/test_operations.py new file mode 100644 index 0000000..1621a74 --- /dev/null +++ b/tests/sml_values/test_operations/test_operations.py @@ -0,0 +1,162 @@ +from typing import Any, Final, Literal +from unittest.mock import Mock + +import pytest +from tests.sml_values.test_operations.helper import check_description, check_operation_repr + +from sml2mqtt.const import SmlFrameValues +from sml2mqtt.sml_value import SmlValue +from sml2mqtt.sml_value.base import ValueOperationBase +from sml2mqtt.sml_value.operations import OffsetOperation, OrOperation, SequenceOperation + + +def test_repr(): + check_operation_repr(OrOperation()) + check_operation_repr(SequenceOperation()) + + +class MockOperationsGroup: + def __init__(self, operation: ValueOperationBase): + self.sentinel = object() + self.operation: Final = operation + self.mocks: list[Mock] = [] + + def assert_called(self, *args: float | Literal['-']): + for mock, arg in zip(self.mocks, args, strict=True): # type: Mock, float | Literal['-'] + if arg == '-': + mock.assert_not_called() + else: + mock.assert_called_once_with(arg, self.sentinel) + + def get_operation_mock(self, return_value: Any) -> Mock: + m = Mock(spec_set=['process_value']) + m.process_value = f = Mock(return_value=return_value) + self.mocks.append(f) + return m + + def process_value(self, value: float): + if isinstance(self.operation, SmlValue): + m = Mock() + m.get_value = Mock(return_value=value) + m.obis = 'obis' + ret = self.operation.process_frame(SmlFrameValues.create(0, [m])) + self.sentinel = self.mocks[0].call_args[0][1] + return ret + + return self.operation.process_value(value, self.sentinel) + + +def get_mock_group(cls: type[OrOperation | SequenceOperation | SmlValue], *return_values: Any) -> MockOperationsGroup: + + c = cls() if cls is not SmlValue else cls('obis', Mock()) + m = MockOperationsGroup(c) + for return_value in return_values: + c.add_operation(m.get_operation_mock(return_value)) + return m + + +def test_or_no_exit(): + m = get_mock_group(OrOperation, None, None, None) + assert m.process_value(1) is None + m.assert_called(1, 1, 1) + + +def test_or_last_exit(): + m = get_mock_group(OrOperation, None, None, 5) + assert m.process_value(1) == 5 + m.assert_called(1, 1, 1) + + +def test_or_first_exit(): + m = get_mock_group(OrOperation, 3, 99, 77) + assert m.process_value(1) == 3 + m.assert_called(1, 1, 1) + + +def test_or_single(): + m = get_mock_group(OrOperation, None) + assert m.process_value(1) is None + m.assert_called(1) + + m = get_mock_group(OrOperation, 11) + assert m.process_value(1) == 11 + m.assert_called(1) + + +def test_or_description(): + o = OrOperation() + o.add_operation(OffsetOperation(3)) + + check_description( + o, [ + '- Or:', + ' - Offset: 3', + ] + ) + + o.add_operation(OffsetOperation(9)) + + check_description( + o, [ + '- Or:', + ' - Offset: 3', + ' - Offset: 9', + ] + ) + + assert o.process_value(0, None) == 3 + assert o.process_value(None, None) is None + + +@pytest.mark.parametrize('cls', [SequenceOperation, SmlValue]) +def test_seq_no_exit(cls): + m = get_mock_group(cls, 1, 2, 3) + assert m.process_value(0) == 3 + m.assert_called(0, 1, 2) + + +@pytest.mark.parametrize('cls', [SequenceOperation, SmlValue]) +def test_seq_first_exit(cls): + m = get_mock_group(cls, None, 2, None) + assert m.process_value(1) is None + m.assert_called(1, None, 2) + + +@pytest.mark.parametrize('cls', [SequenceOperation, SmlValue]) +def test_seq_last_exit(cls): + m = get_mock_group(cls, 1, 2, None) + assert m.process_value(0) is None + m.assert_called(0, 1, 2) + + +@pytest.mark.parametrize('cls', [SequenceOperation, SmlValue]) +def test_seq_single(cls): + m = get_mock_group(cls, None) + assert m.process_value(1) is None + m.assert_called(1) + + m = get_mock_group(SequenceOperation, 11) + assert m.process_value(1) == 11 + m.assert_called(1) + + +def test_sequence_description(): + o = SequenceOperation() + o.add_operation(OffsetOperation(3)) + + check_description( + o, [ + '- Sequence:', + ' - Offset: 3', + ] + ) + + o.add_operation(OffsetOperation(9)) + + check_description( + o, [ + '- Sequence:', + ' - Offset: 3', + ' - Offset: 9', + ] + ) diff --git a/tests/sml_values/test_operations/test_time_series.py b/tests/sml_values/test_operations/test_time_series.py new file mode 100644 index 0000000..766b97b --- /dev/null +++ b/tests/sml_values/test_operations/test_time_series.py @@ -0,0 +1,109 @@ +from tests.sml_values.test_operations.helper import check_description, check_operation_repr + +from sml2mqtt.const import SmlFrameValues, TimeSeries +from sml2mqtt.sml_value.base import SmlValueInfo +from sml2mqtt.sml_value.operations import ( + MaxOfIntervalOperation, + MeanOfIntervalOperation, + MinOfIntervalOperation, +) + + +def info(timestamp: int): + return SmlValueInfo(None, SmlFrameValues.create(timestamp, []), 0) + + +def test_max(): + o = MaxOfIntervalOperation(TimeSeries(5), False) + check_operation_repr(o, 'interval=5s') + + assert o.process_value(None, info(0)) is None + assert o.process_value(5, info(0)) == 5 + assert o.process_value(3, info(3)) == 5 + assert o.process_value(2, info(8)) == 3 + assert o.process_value(1, info(13)) == 2 + assert o.process_value(1, info(14)) == 2 + + o = MaxOfIntervalOperation(TimeSeries(5, wait_for_data=True), False) + check_operation_repr(o, 'interval=5s') + + assert o.process_value(None, info(0)) is None + assert o.process_value(5, info(0)) is None + assert o.process_value(3, info(3)) is None + assert o.process_value(2, info(8)) == 3 + assert o.process_value(1, info(13)) == 2 + assert o.process_value(1, info(14)) == 2 + + o = MaxOfIntervalOperation(TimeSeries(5, wait_for_data=True), True) + check_operation_repr(o, 'interval=5s') + + assert o.process_value(None, info(0)) is None + assert o.process_value(5, info(0)) is None + assert o.process_value(3, info(3)) is None + assert o.process_value(2, info(8)) == 3 + assert o.process_value(2, info(10)) is None + assert o.process_value(1, info(14)) is None + assert o.process_value(1, info(15)) == 2 + + check_description( + o, [ + '- Max Of Interval:', + ' Interval: 5 seconds', + ' Wait for data: True', + ' Reset after value: True', + ] + ) + + +def test_min(): + o = MinOfIntervalOperation(TimeSeries(5), False) + check_operation_repr(o, 'interval=5s') + + assert o.process_value(None, info(0)) is None + assert o.process_value(1, info(0)) == 1 + assert o.process_value(2, info(3)) == 1 + assert o.process_value(3, info(8)) == 2 + assert o.process_value(4, info(13)) == 3 + assert o.process_value(5, info(14)) == 3 + + check_description( + o, [ + '- Min Of Interval:', + ' Interval: 5 seconds', + ' Wait for data: False', + ' Reset after value: False', + ] + ) + + +def test_mean(): + o = MeanOfIntervalOperation(TimeSeries(10), False) + check_operation_repr(o, 'interval=10s') + + assert o.process_value(None, info(0)) is None + assert o.process_value(1, info(0)) is None + assert o.process_value(2, info(5)) == 1 + assert o.process_value(2, info(10)) == 1.5 + assert o.process_value(2, info(13)) == 1.8 + assert o.process_value(4, info(15)) == 2 + assert o.process_value(4, info(20)) == 3 + + o = MeanOfIntervalOperation(TimeSeries(10, wait_for_data=True), False) + check_operation_repr(o, 'interval=10s') + + assert o.process_value(None, info(0)) is None + assert o.process_value(1, info(0)) is None + assert o.process_value(2, info(5)) is None + assert o.process_value(2, info(10)) == 1.5 + assert o.process_value(2, info(13)) == 1.8 + assert o.process_value(4, info(15)) == 2 + assert o.process_value(4, info(20)) == 3 + + check_description( + o, [ + '- Mean Of Interval:', + ' Interval: 10 seconds', + ' Wait for data: True', + ' Reset after value: False', + ] + ) diff --git a/tests/sml_values/test_operations/test_workarounds.py b/tests/sml_values/test_operations/test_workarounds.py new file mode 100644 index 0000000..b9d4af0 --- /dev/null +++ b/tests/sml_values/test_operations/test_workarounds.py @@ -0,0 +1,62 @@ +from smllib.sml import SmlListEntry +from tests.sml_values.test_operations.helper import check_description, check_operation_repr + +from sml2mqtt.const import SmlFrameValues +from sml2mqtt.sml_value.base import SmlValueInfo +from sml2mqtt.sml_value.operations import NegativeOnEnergyMeterWorkaroundOperation + + +def get_info() -> SmlValueInfo: + + power = SmlListEntry() + power.obis = '01000f0700ff' + power.unit = 27 + power.scaler = -1 + power.value = 41890 + + energy = SmlListEntry() + energy.obis = '0100010800ff' + energy.status = 0x1A2 + energy.unit = 30 + energy.scaler = -1 + energy.value = 300371964 + + return SmlValueInfo(power, SmlFrameValues.create(0, [power, energy]), 0) + + +def test_repr(): + o = NegativeOnEnergyMeterWorkaroundOperation() + check_operation_repr(o, '0100010800ff') + + +def test_none(): + o = NegativeOnEnergyMeterWorkaroundOperation() + assert o.process_value(None, get_info()) is None + + +def test_make_negative(): + o = NegativeOnEnergyMeterWorkaroundOperation() + + info = get_info() + value = info.value.get_value() + + assert o.process_value(value, info) == -4189.0 + + +def test_keep_positive(): + o = NegativeOnEnergyMeterWorkaroundOperation() + + info = get_info() + value = info.value.get_value() + + energy = info.frame.get_value('0100010800ff') + energy.status = 0x182 + + assert o.process_value(value, info) == 4189.0 + + +def test_description(): + check_description( + NegativeOnEnergyMeterWorkaroundOperation(), + '- Negative On Status Of Energy Meter 0100010800ff' + ) diff --git a/tests/sml_values/test_setup_operations.py b/tests/sml_values/test_setup_operations.py new file mode 100644 index 0000000..f59bc8f --- /dev/null +++ b/tests/sml_values/test_setup_operations.py @@ -0,0 +1,150 @@ +import inspect +import types +from datetime import time +from typing import Annotated, Union, get_args, get_origin + +import pytest +from pydantic import BaseModel + +import sml2mqtt.config.operations as operations_module +from sml2mqtt.config.operations import Offset, OperationsModels, Sequence +from sml2mqtt.sml_value.operations import OffsetOperation, SequenceOperation, VirtualMeterOperation +from sml2mqtt.sml_value.setup_operations import MAPPING, get_kwargs_names, setup_operations + + +def assert_origins_equal(a, b): + if a is types.UnionType and b is Union: + return None + if b is types.UnionType and a is Union: + return None + + assert a is b + + +def remove_annotated(obj): + ret = [] + for sub in get_args(obj): + if get_origin(sub) is Annotated: + sub = get_args(sub)[0] + ret.append(sub) + return ret + + +def get_kwargs_return_annotation(obj) -> None | str: + name = 'get_kwargs' + if not hasattr(obj, name): + raise ValueError() + + if (sig := inspect.signature(getattr(obj, name)).return_annotation) == 'None': + return None + return sig + + +@pytest.mark.parametrize(('config_model', 'operation'), tuple(MAPPING.items())) +def test_field_to_init(config_model: type[BaseModel], operation: callable): + config_model.model_rebuild() + + sig_o = inspect.signature(operation) + params = sig_o.parameters + + config_provides = {} + + if kwarg_func_names := get_kwargs_names(config_model): + for kwarg_func_name in kwarg_func_names: + return_annotation = inspect.signature(getattr(config_model, kwarg_func_name)).return_annotation + typed_dict = getattr(operations_module, return_annotation) + annotations = inspect.get_annotations(typed_dict) + + for name, fwd_ref in annotations.items(): + ref_type = fwd_ref._evaluate(vars(operations_module), {}, frozenset()) + assert name not in config_provides, config_provides + config_provides[name] = ref_type + else: + for _cfg_name, _cfg_field in config_model.model_fields.items(): + config_provides[_cfg_name] = _cfg_field.annotation + + for name, type_hint in config_provides.items(): + if name == 'type': + continue + + assert name in params + param = params[name] + + if origin_cfg := get_origin(type_hint): + origin_op = get_origin(param.annotation) + assert_origins_equal(origin_cfg, origin_op) + + args_cfg = remove_annotated(type_hint) + args_op = remove_annotated(param.annotation) + assert args_cfg == args_op + + else: + param_hint = param.annotation + assert type_hint == param_hint or type_hint.__name__ == param_hint + + if missing := set(params) - set(config_provides): + msg = f'The following arguments are missing for {operation.__name__}: {", ".join(missing)}' + raise ValueError(msg) + + +def test_all_models_in_mapping(): + if missing := set(OperationsModels) - set(MAPPING): + msg = f'Missing in OperationsModels: {", ".join(m.__name__ for m in missing)}' + raise ValueError(msg) + + if missing := set(MAPPING) - set(OperationsModels): + msg = f'Missing in MAPPING: {", ".join(m.__name__ for m in missing)}' + raise ValueError(msg) + + +def test_simple(): + cfg = Sequence(sequence=[ + Offset(offset=5) + ]) + + o = SequenceOperation() + + setup_operations(o, cfg) + + assert list(o.describe()) == [ + '- Sequence:', + ' - Offset: 5', + ] + + +def test_virtual_meter(): + + cfg = Sequence(sequence=[ + {'type': 'meter', 'start now': True, 'reset times': ['02:00'], 'reset days': ['mon', 6]}, + ]) + + o = SequenceOperation() + + setup_operations(o, cfg) + + assert len(o.operations) == 1 + + o = o.operations[0] + assert isinstance(o, VirtualMeterOperation) + assert o._dt_finder.times == (time(2, 0), ) + assert o._dt_finder.days == (6, ) + assert o._dt_finder.dows == (1, ) + + +def test_complex(): + cfg = Sequence(sequence=[ + {'offset': 5}, + {'or': [{'offset': 5}, {'factor': 3}]} + ]) + + o = SequenceOperation() + + setup_operations(o, cfg) + + assert list(o.describe()) == [ + '- Sequence:', + ' - Offset: 5', + ' - Or:', + ' - Offset: 5', + ' - Factor: 3' + ] diff --git a/tests/sml_values/test_values.py b/tests/sml_values/test_values.py new file mode 100644 index 0000000..e2182bb --- /dev/null +++ b/tests/sml_values/test_values.py @@ -0,0 +1,121 @@ +import logging + +import pytest + +from sml2mqtt.const import SmlFrameValues +from sml2mqtt.errors import ( + RequiredObisValueNotInFrameError, + Sml2MqttExceptionWithLog, + UnprocessedObisValuesReceivedError, +) +from sml2mqtt.mqtt import MqttObj +from sml2mqtt.sml_value import SmlValue, SmlValues +from sml2mqtt.sml_value.operations import OnChangeFilterOperation +from sml_values.test_operations.helper import check_description + + +def test_values(sml_frame_1_values: SmlFrameValues, no_mqtt): + mqtt = MqttObj(topic_fragment='test', qos=0, retain=False).update() + + v = SmlValues() + v.set_skipped('010060320101', '0100600100ff', '0100020800ff') + + v.add_value( + SmlValue('0100010800ff', mqtt.create_child('energy')).add_operation(OnChangeFilterOperation()) + ) + v.add_value( + SmlValue('0100100700ff', mqtt.create_child('power')).add_operation(OnChangeFilterOperation()) + ) + + # The change filter prevents a republish + for _ in range(10): + v.process_frame(sml_frame_1_values) + assert no_mqtt == [('test/energy', 253917.7, 0, False), ('test/power', 272, 0, False)] + + # test description + check_description(v, [ + 'Skipped: 0100020800ff, 0100600100ff, 010060320101', + '', + '', + ' obis : 0100010800ff', + ' topic: test/energy', + ' operations:', + ' - On Change Filter', + '', + '', + ' obis : 0100100700ff', + ' topic: test/power', + ' operations:', + ' - On Change Filter', + '', + ]) + + +def get_error_message(e: Sml2MqttExceptionWithLog, caplog) -> list[str]: + e.log_msg(logging.getLogger('test')) + + msgs = [] + for rec_tuple in caplog.record_tuples: + name, level, msg = rec_tuple + assert name == 'test' + assert level == logging.ERROR + msgs.append(msg) + + return msgs + + +@pytest.mark.ignore_log_errors() +def test_too_much(sml_frame_1_values: SmlFrameValues, no_mqtt, caplog): + v = SmlValues() + v.set_skipped('010060320101', '0100600100ff') + + v.add_value( + SmlValue('0100010800ff', MqttObj()).add_operation(OnChangeFilterOperation()) + ) + v.add_value( + SmlValue('0100100700ff', MqttObj()).add_operation(OnChangeFilterOperation()) + ) + + with pytest.raises(UnprocessedObisValuesReceivedError) as e: + v.process_frame(sml_frame_1_values) + + assert get_error_message(e.value, caplog) == [ + 'Unexpected obis id received!', + '', + ' obis : 0100020800ff (1-0:2.8.0*255)', + ' status : None', + ' val_time : None', + ' unit : 30', + ' scaler : -1', + ' value : 0', + ' value_signature: None', + ' -> 0.0Wh (Zählerstand Einspeisung Total)' + ] + + +@pytest.mark.ignore_log_errors() +def test_missing(sml_frame_1_values: SmlFrameValues, no_mqtt, caplog): + v = SmlValues() + v.set_skipped('010060320101', '0100600100ff', '0100020800ff', '0100010800ff', '0100100700ff') + + v.add_value( + SmlValue('1100010800ff', MqttObj()).add_operation(OnChangeFilterOperation()) + ) + + with pytest.raises(RequiredObisValueNotInFrameError) as e: + v.process_frame(sml_frame_1_values) + + assert get_error_message(e.value, caplog) == ['Expected obis id missing in frame: 1100010800ff!'] + + # Now two values are missing + v.add_value( + SmlValue('1200010800ff', MqttObj()).add_operation(OnChangeFilterOperation()) + ) + + with pytest.raises(RequiredObisValueNotInFrameError) as e: + v.process_frame(sml_frame_1_values) + + assert get_error_message(e.value, caplog) == [ + 'Expected obis id missing in frame: 1100010800ff!', + 'Expected obis ids missing in frame: 1100010800ff, 1200010800ff!' + ] diff --git a/tests/test_docs.py b/tests/test_docs.py index 09feee2..7211877 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -1,78 +1,156 @@ +from collections.abc import Callable +from dataclasses import dataclass from inspect import getmembers, isclass from pathlib import Path +from typing import Any +from _pytest.monkeypatch import derive_importpath from easyconfig import yaml from pydantic import BaseModel import sml2mqtt +from sml2mqtt.config.inputs import SmlSourceSettingsBase +from sml2mqtt.config.operations import HasDateTimeFields, HasIntervalFields +from sml2mqtt.sml_value.setup_operations import MAPPING, setup_operations -def test_sample_yaml(pytestconfig): - file = pytestconfig.rootpath / 'docs' / 'configuration.rst' +@dataclass +class YamlBlock: + file: Path + line_no: int + model: str + lines: list[str] - all_cfgs = [] + def validate(self, func: Callable[[BaseModel],Any] | None = None): + sample_cfg = '\n'.join(self.lines) - lines = [] - add = False - indent = 0 + target, name = derive_importpath(self.model, True) + model_cls = getattr(name, target) # type: type[BaseModel] - for line in file.read_text().splitlines(): - line = line - stripped = line.strip() + try: + yaml_obj = yaml.yaml_rt.load(sample_cfg) + model = model_cls.model_validate(yaml_obj) + if func: + func(model) + except Exception: + print('') + print(f'Error in {self.file.parent.name}/{self.file.name}:{self.line_no}') + raise + + +def validate_yaml_blocks(file: Path, prefix_model: str = 'yamlmodel: ', func: Callable[[BaseModel],Any] | None = None): + + current_module = '' + + model = '' + obj: YamlBlock | None = None + indentation = 0 - if add: - if not indent and stripped: - while line[indent] == ' ': - indent += 1 + for line_no, line in enumerate(file.read_text().splitlines()): + line = line + line_low = line.strip().lower() + + if line_low.startswith('.. py:currentmodule::'): + current_module = line.strip()[21:].strip() + + if line_low.startswith(prefix_model): + assert not model + model = line.strip()[len(prefix_model):] + if '.' not in model: + if not current_module: + msg = f'currentmodule not set in {file}:{line_no}' + raise ValueError(msg) + model = f'{current_module}.{model}' + + if obj is not None: + # we need one empty line before the yaml starts + if not obj.lines and line_low: + continue - if stripped and line[0] != ' ': - all_cfgs.append(lines) - add = False + # find out indentation + if not indentation and line_low: + while line[indentation] == ' ': + indentation += 1 + + # we have a non-indented line -> yaml is finished + if line_low and line[0] != ' ': + obj.validate(func) + obj = None + indentation = 0 + model = '' continue - lines.append(line[indent:]) + obj.lines.append(line) - if stripped.startswith('.. code-block:: yaml') or stripped.startswith('.. code-block:: yml'): - add = True - lines = [] + if line_low.startswith(('.. code-block:: yaml', '.. code-block:: yml')): + if not model: + msg = f'Model object not set in {file}:{line_no}' + raise ValueError(msg) + if obj is not None: + msg = f'Object already set in {file}:{line_no}: {obj.model} line {obj.line_no}' + raise ValueError(msg) - if add: - all_cfgs.append(lines) + obj = YamlBlock(file, line_no, model, []) - assert len(all_cfgs) == 2 - for cfg_lines in all_cfgs: - sample_cfg = '\n'.join(cfg_lines) + if obj: + obj.validate(func) - map = yaml.yaml_rt.load(sample_cfg) - sml2mqtt.config.config.Settings(**map) +def test_yaml_samples(pytestconfig): -def test_config_documentation_complete(pytestconfig): - cfg_docs: Path = pytestconfig.rootpath / 'docs' / 'configuration.rst' - cfg_model_dir: Path = pytestconfig.rootpath / 'src' / 'sml2mqtt' / 'config' - assert cfg_model_dir.is_dir() + class DummyOperationParent: + def add_operation(self, obj): + pass - documented_objs = set() + class HasOperationsModel(BaseModel): + operations: list[BaseModel] + + def check_obj(model: BaseModel): + if model.__class__ in MAPPING: + setup_operations(DummyOperationParent(), HasOperationsModel(operations=[model])) + + for file in (pytestconfig.rootpath / 'docs').iterdir(): + if file.suffix.lower() == '.rst': + validate_yaml_blocks(file, func=check_obj) + + +def _get_documented_objs(path: Path, objs: set[str]): - # documented config current_module = '' - for line in (x.strip().replace(' ', '') for x in cfg_docs.read_text().splitlines()): # type: str + + for line in (x.strip().replace(' ', '') for x in path.read_text().splitlines()): # type: str if line.startswith('.. py:currentmodule::'): current_module = line[21:].strip() continue if line.startswith('.. autopydantic_model::'): obj_name = line[23:].strip() - if current_module: + if '.' not in obj_name: + assert current_module obj_name = f'{current_module}.{obj_name}' - assert obj_name not in documented_objs - documented_objs.add(obj_name) + assert obj_name not in objs + objs.add(obj_name) + + +def test_config_documentation_complete(pytestconfig): + cfg_model_dir: Path = pytestconfig.rootpath / 'src' / 'sml2mqtt' / 'config' + assert cfg_model_dir.is_dir() + + documented_objs = set() + _get_documented_objs(pytestconfig.rootpath / 'docs' / 'configuration.rst', documented_objs) + _get_documented_objs(pytestconfig.rootpath / 'docs' / 'operations.rst', documented_objs) # get Config implementation from source existing_objs = set() for module_name in [f.stem for f in cfg_model_dir.glob('**/*.py')]: module = getattr(sml2mqtt.config, module_name) - cfg_objs = [x[1] for x in getmembers(module, lambda x: isclass(x) and issubclass(x, BaseModel))] + cfg_objs = [ + x[1] for x in getmembers( + module, lambda x: isclass(x) and issubclass(x, BaseModel) and x not in ( + SmlSourceSettingsBase, HasIntervalFields, HasDateTimeFields + ) + ) + ] cfg_names = { f'{obj.__module__}.{obj.__qualname__}' for obj in cfg_objs if not obj.__module__.startswith('easyconfig.') } diff --git a/tests/test_mqtt_obj.py b/tests/test_mqtt_obj.py index 0219bd0..86adc0f 100644 --- a/tests/test_mqtt_obj.py +++ b/tests/test_mqtt_obj.py @@ -1,4 +1,6 @@ -from sml2mqtt.mqtt import MqttObj +import pytest + +from sml2mqtt.mqtt import MqttObj, check_for_duplicate_topics def test_topmost(monkeypatch): @@ -14,6 +16,13 @@ def test_topmost(monkeypatch): assert parent.retain is True +def test_prefix_empty(monkeypatch): + parent = MqttObj('', 2, True).update() + child = parent.create_child('child') + + assert (child.topic, child.qos, child.retain) == ('child', 2, True) + + def test_child_change(monkeypatch): parent = MqttObj('base', 2, True).update() child = parent.create_child('child') @@ -41,3 +50,16 @@ def test_child_change(monkeypatch): parent.cfg.retain = False parent.update() assert (child.topic, child.qos, child.retain) == ('base/child', 0, False) + + +@pytest.mark.ignore_log_warnings() +def test_check_for_duplicate_messages(caplog): + parent = MqttObj('base', 2, True).update() + parent.create_child('child') + parent.create_child('child') + + check_for_duplicate_topics(parent) + + msg = "\n".join(x.msg for x in caplog.records) + + assert msg == 'Topic "base/child" is already configured!' diff --git a/tests/values/transformations/__init__.py b/tests/test_source/__init__.py similarity index 100% rename from tests/values/transformations/__init__.py rename to tests/test_source/__init__.py diff --git a/tests/test_source/conftest.py b/tests/test_source/conftest.py new file mode 100644 index 0000000..16c46a7 --- /dev/null +++ b/tests/test_source/conftest.py @@ -0,0 +1,27 @@ +from unittest.mock import Mock + +import pytest + +from sml2mqtt.const import DeviceProto + + +class DeviceMock(DeviceProto): + + def __init__(self): + self.on_source_data = Mock() + self.on_source_failed = Mock() + self.on_error = Mock() + + @property + def name(self) -> str: + return f'DeviceMock at 0x{id(self):x}' + + +@pytest.fixture() +def device_mock() -> DeviceMock: + m = DeviceMock() + m.on_source_data.assert_not_called() + m.on_source_failed.assert_not_called() + m.on_error.assert_not_called() + + return m diff --git a/tests/test_source/test_create.py b/tests/test_source/test_create.py new file mode 100644 index 0000000..15775c9 --- /dev/null +++ b/tests/test_source/test_create.py @@ -0,0 +1,33 @@ +from aiohttp import BasicAuth + +from sml2mqtt.config.inputs import HttpSourceSettings +from sml2mqtt.sml_source import create_source +from sml2mqtt.sml_source.http import HttpSource + + +async def test_create_http_no_auth(device_mock): + cfg = HttpSourceSettings(type='http', url='http://localhost/a', interval=3, timeout=9) + obj = await create_source(device_mock, cfg) + + assert isinstance(obj, HttpSource) + assert obj.url == 'http://localhost/a' + assert obj.auth is None + assert obj.interval == 3 + + device_mock.on_source_data.assert_not_called() + device_mock.on_source_failed.assert_not_called() + device_mock.on_error.assert_not_called() + + +async def test_create_http_auth(device_mock): + cfg = HttpSourceSettings(type='http', url='http://localhost/a', interval=3, timeout=9, user='u', password='p') + obj = await create_source(device_mock, cfg) + + assert isinstance(obj, HttpSource) + assert obj.url == 'http://localhost/a' + assert obj.auth == BasicAuth('u', 'p') + assert obj.interval == 3 + + device_mock.on_source_data.assert_not_called() + device_mock.on_source_failed.assert_not_called() + device_mock.on_error.assert_not_called() diff --git a/tests/test_source/test_http.py b/tests/test_source/test_http.py new file mode 100644 index 0000000..eb9c338 --- /dev/null +++ b/tests/test_source/test_http.py @@ -0,0 +1,101 @@ +import sys +from asyncio import TimeoutError + +import pytest +from aiohttp import ClientTimeout +from aioresponses import aioresponses +from tests.helper import wait_for_call + +from sml2mqtt.errors import HttpStatusError +from sml2mqtt.sml_source.http import HttpSource, close_session + + +@pytest.fixture() +def source(device_mock): + return HttpSource(device_mock, 'http://localhost:39999', interval=0.020, auth=None, timeout=ClientTimeout(0.5)) + + +@pytest.mark.skipif(sys.platform.lower() != "win32", reason="It's a mystery why this fails in CI") +async def test_200(sml_data_1, device_mock, source): + + with aioresponses() as m: + m.get(source.url, body=sml_data_1) + + source.start() + try: + await wait_for_call(device_mock.on_source_data, 1) + finally: + await source.cancel_and_wait() + + device_mock.on_source_data.assert_called_once_with(sml_data_1) + device_mock.on_source_failed.assert_not_called() + device_mock.on_error.assert_not_called() + + await close_session() + + +@pytest.mark.skipif(sys.platform.lower() != "win32", reason="It's a mystery why this fails in CI") +async def test_400_then_200(sml_data_1, device_mock, source): + + with aioresponses() as m: + m.get(source.url, status=404) + m.get(source.url, status=404) + m.get(source.url, body=sml_data_1) + + source.start() + try: + await wait_for_call(device_mock.on_source_data, 1) + finally: + await source.cancel_and_wait() + + device_mock.on_source_data.assert_called_once_with(sml_data_1) + device_mock.on_source_failed.assert_not_called() + device_mock.on_error.assert_not_called() + + await close_session() + + +@pytest.mark.skipif(sys.platform.lower() != "win32", reason="It's a mystery why this fails in CI") +async def test_400(device_mock, source): + + with aioresponses() as m: + for _ in range(10): + m.get(source.url, status=404) + + source.start() + try: + await wait_for_call(device_mock.on_error, 1) + finally: + await source.cancel_and_wait() + + device_mock.on_source_data.assert_not_called() + device_mock.on_source_failed.assert_not_called() + device_mock.on_error.assert_called_once() + + await close_session() + + +def test_error_repr(): + assert str(HttpStatusError(404)) == 'HttpStatusError: 404' + + +@pytest.mark.skipif(sys.platform.lower() != "win32", reason="It's a mystery why this fails in CI") +async def test_timeout(device_mock, source): + + e = TimeoutError() + + with aioresponses() as m: + for _ in range(10): + m.get(source.url, exception=e) + + source.start() + try: + await wait_for_call(device_mock.on_error, 1) + finally: + await source.cancel_and_wait() + + device_mock.on_source_data.assert_not_called() + device_mock.on_source_failed.assert_not_called() + device_mock.on_error.assert_called_once_with(e, show_traceback=False) + + await close_session() diff --git a/tests/values/filters/test_diff.py b/tests/values/filters/test_diff.py deleted file mode 100644 index 2714ad7..0000000 --- a/tests/values/filters/test_diff.py +++ /dev/null @@ -1,29 +0,0 @@ -from sml2mqtt.sml_value.filter import DiffAbsFilter, DiffPercFilter - - -def test_abs(): - f = DiffAbsFilter(5) - assert f.required(0) - f.done(0) - - assert not f.required(1) - assert not f.required(4.999999) - assert f.required(5) - - assert not f.required(-1) - assert not f.required(-4.999999) - assert f.required(-5) - - -def test_perc(): - f = DiffPercFilter(5) - assert f.required(100) - f.done(100) - - assert not f.required(100) - assert not f.required(104.999999) - assert f.required(105) - - assert not f.required(99) - assert not f.required(95.0000001) - assert f.required(95) diff --git a/tests/values/transformations/test_math.py b/tests/values/transformations/test_math.py deleted file mode 100644 index 590dcbd..0000000 --- a/tests/values/transformations/test_math.py +++ /dev/null @@ -1,34 +0,0 @@ -from unittest.mock import Mock - -from smllib.sml import SmlListEntry - -from sml2mqtt.sml_value import SmlValue -from sml2mqtt.sml_value.filter import RefreshEvery -from sml2mqtt.sml_value.transformations import RoundTransformation - - -def test_round(): - r = RoundTransformation(0) - assert r.process(1.11) == 1 - assert isinstance(r.process(1.11), int) - assert str(r) == '' - - r = RoundTransformation(1) - assert r.process(1.11) == 1.1 - assert isinstance(r.process(1.11), float) - assert str(r) == '' - - -def test_round_call(): - m = Mock() - v = SmlValue('device', 'obis', m, [], [RoundTransformation(0)], [RefreshEvery(120)]) - - power = SmlListEntry() - power.obis = '01000f0700ff' - power.unit = 27 - power.scaler = -1 - power.value = 41891 - - v.set_value(power, {}) - - m.publish.assert_called_once_with(4189) diff --git a/tests/values/workarounds/__init__.py b/tests/values/workarounds/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/values/workarounds/test_negative_on_energy_meter.py b/tests/values/workarounds/test_negative_on_energy_meter.py deleted file mode 100644 index 1bded77..0000000 --- a/tests/values/workarounds/test_negative_on_energy_meter.py +++ /dev/null @@ -1,54 +0,0 @@ -from typing import Tuple - -from smllib.sml import SmlListEntry - -from sml2mqtt.sml_value.workarounds import NegativeOnEnergyMeterStatus - - -def get_entries() -> Tuple[SmlListEntry, SmlListEntry]: - - power = SmlListEntry() - power.obis = '01000f0700ff' - power.unit = 27 - power.scaler = -1 - power.value = 41890 - - energy = SmlListEntry() - energy.obis = '0100010800ff' - energy.status = 0x1A2 - energy.unit = 30 - energy.scaler = -1 - energy.value = 300371964 - - return power, energy - - -def test_make_negative(): - - wk = NegativeOnEnergyMeterStatus(True) - - power, energy = get_entries() - assert energy.status == 418 - - wk.fix(power, {power.obis: power, energy.obis: energy}) - - assert power.value == -41890 - assert power.obis == '01000f0700ff' - assert power.unit == 27 - assert power.scaler == -1 - - -def test_keep_positive(): - - wk = NegativeOnEnergyMeterStatus(True) - - power, energy = get_entries() - energy.status = 0x182 - assert energy.status == 386 - - wk.fix(power, {power.obis: power, energy.obis: energy}) - - assert power.value == 41890 - assert power.obis == '01000f0700ff' - assert power.unit == 27 - assert power.scaler == -1 diff --git a/tox.ini b/tox.ini index 3618d8b..374d97b 100644 --- a/tox.ini +++ b/tox.ini @@ -1,19 +1,17 @@ # content of: tox.ini , put in same dir as setup.py [tox] envlist = - py38 - py39 py310 py311 + py312 docs [gh-actions] python = - 3.8: py38 - 3.9: py39 3.10: py310, docs 3.11: py311 + 3.12: py312 [testenv] deps = @@ -23,10 +21,6 @@ commands = python -m pytest -[pytest] -asyncio_mode = auto - - [testenv:docs] description = invoke sphinx-build to build the HTML docs @@ -39,3 +33,12 @@ commands = sphinx-build -d "{envtmpdir}{/}doctree" docs "{toxworkdir}{/}docs_out" --color -b html -E -W -n --keep-going allowlist_externals = mkdir + + + +[pytest] +asyncio_mode = auto + +markers = + ignore_log_errors: Ignore logged errors + ignore_log_warnings: Ignore logged warnings diff --git a/whitelist.txt b/whitelist.txt deleted file mode 100644 index 388fd74..0000000 --- a/whitelist.txt +++ /dev/null @@ -1,31 +0,0 @@ -mqtt -qos - -# sml -sml -smllib -scaler -sml2mqtt - -# python packages -pydantic -easyconfig -unittest -binascii - -# pyserial -baudrate -stopbits -bytesize - -# pytest -autouse -caplog - -# pydantic -conint -constr -validator - -# own words -cfg