From 843d4366de3708308be01be64d3e4c8b63faba35 Mon Sep 17 00:00:00 2001 From: cefili Date: Tue, 14 Nov 2023 13:13:14 +0100 Subject: [PATCH] Added IoT class. --- doc/best_practices.md | 3 + python-lib/tc_etl_lib/README.md | 21 ++ python-lib/tc_etl_lib/tc_etl_lib/__init__.py | 4 +- python-lib/tc_etl_lib/tc_etl_lib/cb.py | 29 +-- .../tc_etl_lib/tc_etl_lib/exceptions.py | 50 ++++ python-lib/tc_etl_lib/tc_etl_lib/iot.py | 102 ++++++++ python-lib/tc_etl_lib/tc_etl_lib/test_iot.py | 223 ++++++++++++++++++ 7 files changed, 404 insertions(+), 28 deletions(-) create mode 100644 python-lib/tc_etl_lib/tc_etl_lib/exceptions.py create mode 100644 python-lib/tc_etl_lib/tc_etl_lib/iot.py create mode 100644 python-lib/tc_etl_lib/tc_etl_lib/test_iot.py diff --git a/doc/best_practices.md b/doc/best_practices.md index c93a501..616794d 100644 --- a/doc/best_practices.md +++ b/doc/best_practices.md @@ -48,6 +48,9 @@ Las funciones que actualmente soporta la librería son: - modulo `cb`: Módulo que incluye funciones relacionadas con la comunicación con el Context Broker. - `send_batch`: Función que envía un lote de entidades al Context Broker. Recibe un listado con todos los tokens por subservicio y usa el correspondiente para realizar la llamada al Context Broker. Si no se dispone de token o ha caducado, se solicita o renueva el token según el caso y luego envía los datos. - `get_entities_page`: Función que permite la recogida de datos del Context Broker. Permite el uso de ciertos parámetros como offset, limit, orderBy y type para filtrar la recogida de datos. + - modulo `iot`: Módulo que incluye funciones relacionadas con la comunicación del IoT Agent. + - `send_json`: Función que envía un JSON al IoT Agent. Recibe el nombre del sensor, su API key correspondiente, la URL del servidor y los datos a enviar. + - `send_batch`: Función que envía una lista de diccionarios al IoT Agent. Los envía uno por uno. Recibe el nombre del sensor, su API key correspondiente, la URL del servidor, el tiempo de espera en segundos entre envío y envío y los datos a enviar. Se puede encontrar más detalles de la librería en la documentación de esta. [Ref.](../python-lib/tc_etl_lib/README.md) diff --git a/python-lib/tc_etl_lib/README.md b/python-lib/tc_etl_lib/README.md index 82b3d22..b15738c 100644 --- a/python-lib/tc_etl_lib/README.md +++ b/python-lib/tc_etl_lib/README.md @@ -343,6 +343,27 @@ La librería está creada con diferentes clases dependiendo de la funcionalidad - Reemplaza todos los espacios en blanco consecutivos por el carácter de reemplazo. - NOTA: Esta función no recorta la longitud de la cadena devuelta a 256 caracteres, porque el llamante puede querer conservar la cadena entera para por ejemplo guardarla en algún otro atributo, antes de truncarla. +- Clase `IoT`: En esta clase están las funciones relacionadas con el IoT Agent. + + - `__init__`: constructor de objetos de la clase. + - `send_json`: Función que envía un archivo en formato JSON al IoT Agent. + - :param obligatorio `sensor_name`: El nombre del sensor. + - :param obligatorio `api_key`: La API key correspondiente al sensor. + - :param obligatorio `req_url`: La URL del servicio al que se le quiere enviar los datos. + - :param obligatorio: `data`: Datos a enviar. La estructura debe tener pares de elementos clave-valor (diccionario). + - :raises ValueError: Se lanza cuando los datos a enviar son distintos a un único diccionario. + - :raises Excepction: Se lanza una excepción ConnectionError cuando no puede conectarse al servidor. Se lanza una excepción FetchError cuando se produce un error en en la solicitud HTTP. + - :return: True si el envío de datos es exitoso. + - `send_batch`: Función que envía un archivo en formato JSON al IoT Agent. + - :param obligatorio `sensor_name`: El nombre del sensor. + - :param obligatorio `api_key`: La API key correspondiente al sensor. + - :param obligatorio `req_url`: La URL del servicio al que se le quiere enviar los datos. + - :param obligatorio: `time_sleep`: Es el tiempo de espera entre cada envío de datos en segundos. + - :param obligatorio: `data`: Datos a enviar. La estructura debe tener pares de elementos clave-valor (diccionario).t + - :raises ValueError: Se lanza cuando el tipo de los datos a enviar es incorrecto. + - :raises Excepction: Se lanza una excepción ConnectionError cuando no puede conectarse al servidor. Se lanza una excepción FetchError cuando se produce un error en en la solicitud HTTP. + - :return: True si el envío de datos es exitoso. + Algunos ejemplos de uso de `normalizer`: ``` diff --git a/python-lib/tc_etl_lib/tc_etl_lib/__init__.py b/python-lib/tc_etl_lib/tc_etl_lib/__init__.py index 614d69e..695ee29 100644 --- a/python-lib/tc_etl_lib/tc_etl_lib/__init__.py +++ b/python-lib/tc_etl_lib/tc_etl_lib/__init__.py @@ -20,6 +20,8 @@ # from .auth import authManager -from .cb import FetchError, cbManager +from .cb import cbManager +from .exceptions import FetchError +from .iot import IoT from .store import Store, orionStore, sqlFileStore from .normalizer import normalizer diff --git a/python-lib/tc_etl_lib/tc_etl_lib/cb.py b/python-lib/tc_etl_lib/tc_etl_lib/cb.py index 39f9946..29cc07c 100644 --- a/python-lib/tc_etl_lib/tc_etl_lib/cb.py +++ b/python-lib/tc_etl_lib/tc_etl_lib/cb.py @@ -34,7 +34,7 @@ import time import json -from . import authManager +from . import authManager, exceptions # control urllib3 post and get verify in false import urllib3, urllib3.exceptions @@ -42,31 +42,6 @@ logger = logging.getLogger(__name__) -class FetchError(Exception): - """ - FetchError encapsulates all parameters of an HTTP request and the erroneous response - """ - - response: requests.Response - method: str - url: str - params: Optional[Any] = None - headers: Optional[Any] = None - body: Optional[Any] = None - - def __init__(self, response: requests.Response, method: str, url: str, params: Optional[Any] = None, headers: Optional[Any] = None, body: Optional[Any] = None): - """Constructor for FetchError class""" - self.response = response - self.method = method - self.url = url - self.params = params - self.headers = headers - self.body = body - - def __str__(self) -> str: - return f"Failed to {self.method} {self.url} (headers: {self.headers}, params: {self.params}, body: {self.body}): [{self.response.status_code}] {self.response.text}" - - class cbManager: """ContextBroker Manager @@ -260,7 +235,7 @@ def get_entities_page(self, *, service: Optional[str] = None, subservice: Option respjson = resp.json() logger.error(f'{respjson["name"]}: {respjson["message"]}') if resp.status_code < 200 or resp.status_code > 204: - raise FetchError(response=resp, method="GET", url=req_url, params=params, headers=headers) + raise exceptions.FetchError(response=resp, method="GET", url=req_url, params=params, headers=headers) return resp.json() diff --git a/python-lib/tc_etl_lib/tc_etl_lib/exceptions.py b/python-lib/tc_etl_lib/tc_etl_lib/exceptions.py new file mode 100644 index 0000000..7ec05a3 --- /dev/null +++ b/python-lib/tc_etl_lib/tc_etl_lib/exceptions.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright 2022 Telefónica Soluciones de Informática y Comunicaciones de España, S.A.U. +# +# This file is part of tc_etl_lib +# +# tc_etl_lib is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# tc_etl_lib is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with IoT orchestrator. If not, see http://www.gnu.org/licenses/. + +""" +Exceptions handling for Python. +""" + +import requests +from typing import Any, Optional + +class FetchError(Exception): + """ + FetchError encapsulates all parameters of an HTTP request and the erroneous response. + """ + + response: requests.Response + method: str + url: str + params: Optional[Any] = None + headers: Optional[Any] = None + body: Optional[Any] = None + + def __init__(self, response: requests.Response, method: str, url: str, params: Optional[Any] = None, headers: Optional[Any] = None, body: Optional[Any] = None): + """Constructor for FetchError class.""" + self.response = response + self.method = method + self.url = url + self.params = params + self.headers = headers + self.body = body + + def __str__(self) -> str: + return f"Failed to {self.method} {self.url} (headers: {self.headers}, params: {self.params}, body: {self.body}): [{self.response.status_code}] {self.response.text}" \ No newline at end of file diff --git a/python-lib/tc_etl_lib/tc_etl_lib/iot.py b/python-lib/tc_etl_lib/tc_etl_lib/iot.py new file mode 100644 index 0000000..342fee4 --- /dev/null +++ b/python-lib/tc_etl_lib/tc_etl_lib/iot.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright 2022 Telefónica Soluciones de Informática y Comunicaciones de España, S.A.U. +# +# This file is part of tc_etl_lib +# +# tc_etl_lib is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# tc_etl_lib is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with IoT orchestrator. If not, see http://www.gnu.org/licenses/. + +''' +IoT routines for Python: + - iot.send_json + - iot.send_batch +''' + +from . import exceptions +import requests +import tc_etl_lib as tc +import time +from typing import Any, Iterable + + +class IoT: + """IoT is a class that allows us to communicate with the IoT Agent.""" + + def __init__(self): + pass + + def send_json(self, + sensor_name: str, + api_key: str, + req_url: str, + data: Any) -> None: + params = { + 'i': sensor_name, + 'k': api_key + } + headers = { + "Content-Type": "application/json" + } + + try: + # Verify if data is a single dictionary. + if isinstance(data, dict): + resp = requests.post(url=req_url, json=data, + params=params, headers=headers) + if resp.status_code == 200: + return True + else: + raise exceptions.FetchError( + response=resp, + method="POST", + url=req_url, + params=params, + headers=headers) + else: + raise ValueError( + "The parameter 'data' should be a single dictionary {}.") + except requests.exceptions.ConnectionError as e: + raise e + + def send_batch(self, + sensor_name: str, + api_key: str, + req_url: str, + time_sleep: float, + data: Iterable[Any]) -> None: + params = { + 'i': sensor_name, + 'k': api_key + } + headers = { + "Content-Type": "application/json" + } + + try: + for i in range(0, len(data)): + resp = requests.post( + url=req_url, json=data[i], params=params, headers=headers) + + if resp.status_code == 200: + time.sleep(time_sleep) + return True + else: + raise exceptions.FetchError(response=resp, + method="POST", + url=req_url, + params=params, + headers=headers) + except requests.exceptions.ConnectionError as e: + raise e \ No newline at end of file diff --git a/python-lib/tc_etl_lib/tc_etl_lib/test_iot.py b/python-lib/tc_etl_lib/tc_etl_lib/test_iot.py new file mode 100644 index 0000000..e1c7033 --- /dev/null +++ b/python-lib/tc_etl_lib/tc_etl_lib/test_iot.py @@ -0,0 +1,223 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2023 Telefónica Soluciones de Informática y Comunicaciones de España, S.A.U. +# +# This file is part of tc_etl_lib +# +# tc_etl_lib is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# tc_etl_lib is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with IoT orchestrator. If not, see http://www.gnu.org/licenses/. + +''' +IoT tests +''' + +from . import IoT, exceptions +import pytest +import requests +import unittest +from unittest.mock import patch, Mock + + +class TestIoT(unittest.TestCase): + def test_send_json_success(self): + """A success message should be displayed when + HTTP request is executed successfully.""" + iot = IoT() + with patch('requests.post') as mock_post: + fake_response = Mock() + # Simulates a successful code status. + fake_response.status_code = 200 + mock_post.return_value = fake_response + resp = iot.send_json("fake_sensor", "fake_api_key", + "http://fakeurl.com", {"key": "value"}) + assert resp == True + + + def test_send_json_connection_error(self): + """Should raise an exception if there is a server connection error.""" + iot = IoT() + with patch('requests.post') as mock_post: + mock_post.side_effect = requests.exceptions.ConnectionError() + with pytest.raises(requests.exceptions.ConnectionError): + iot.send_json( + "fake_sensor_name", + "fake_api_key", + "http://fakeurl.com", + {"key": "value"}) + + + def test_send_json_invalid_data_type(self): + """Should raise an exception if the data type is icorrect.""" + iot = IoT() + with pytest.raises(ValueError) as exc_info: + iot.send_json("fake_sensor_name", + "fake_api_key", + "http://fakeurl.com", + ["data"]) + exception_message = str(exc_info.value) + assert "The parameter 'data' should be a single dictionary {}." in str( + exception_message) + + + def test_send_json_set_not_unique(self): + """Should raise an exception if the data is an array of dictionaries.""" + iot = IoT() + with pytest.raises(ValueError) as exc_info: + iot.send_json("fake_sensor_name", "fake_api_key", + "http://fakeurl.com", + [ + {"key_1": "value_1"}, + {"key_2": "value_2"}]) + exception_message = str(exc_info.value) + assert "The parameter 'data' should be a single dictionary {}." in str( + exception_message) + + + def test_send_json_unauthorized(self): + """Should raise an exception if the request is unauthorized.""" + iot = IoT() + with patch('requests.post') as mock_post: + mock_post.return_value.status_code = 401 + with pytest.raises(exceptions.FetchError) as exc_info: + iot.send_json("fake_sensor_name", + "fake_api_key", + "http://fakeurl.com", + {"key": "value"}) + exception_raised = str(exc_info.value) + assert "401" in exception_raised + assert "Failed to POST http://fakeurl.com" in exception_raised + + + def test_send_json_not_found(self): + """Should raise an exception if the request is not found.""" + iot = IoT() + with patch('requests.post') as mock_post: + mock_post.return_value.status_code = 404 + with pytest.raises(exceptions.FetchError) as exc_info: + iot.send_json("fake_sensor_name", "fake_api_key", + "http://fakeurl.com", {"key": "value"}) + + exception_raised = str(exc_info.value) + assert "404" in exception_raised + assert "Failed to POST http://fakeurl.com" in exception_raised + + + def test_send_json_server_error(self): + """Should raise an exception if there is a server error.""" + iot = IoT() + with patch('requests.post') as mock_post: + mock_post.return_value.status_code = 500 + with pytest.raises(exceptions.FetchError) as exc_info: + iot.send_json("fake_sensor_name", + "fake_api_key", + "http://fakeurl.com", + {"key": "value"}) + + exception_raised = str(exc_info.value) + assert "500" in exception_raised + assert "Failed to POST http://fakeurl.com" in exception_raised + + + def test_send_batch_success(self): + """The status code should be 200 if a request is success.""" + iot = IoT() + with patch('requests.post') as mock_post: + fake_response = Mock() + # Simulates a successful status code. + fake_response.status_code = 200 + mock_post.return_value = fake_response + resp = iot.send_batch("fake_sensor", + "fake_api_key", + "http://fakeurl.com", + 0.25, + [ + {"key_1": "value_1"}, + {"key_2", "value_2"} + ]) + + + assert resp == True + + + def test_send_batch_connection_error(self): + """Should raise an exception if there is a connection error.""" + iot = IoT() + with patch('requests.post') as mock_post: + mock_post.side_effect = requests.exceptions.ConnectionError() + with pytest.raises(requests.exceptions.ConnectionError): + iot.send_batch( + "fake_sensor_name", + "fake_api_key", + "http://fakeurl.com", + 0.25, + [ + {"key_1": "value_1"}, + {"key_2": "value_2"} + ]) + + + + def test_send_batch_unauthorized(self): + """Should raise an exception if the request is unauthorized.""" + iot = IoT() + with patch('requests.post') as mock_post: + mock_post.return_value.status_code = 401 + with pytest.raises(exceptions.FetchError) as exc_info: + iot.send_batch("fake_sensor_name", + "fake_api_key", + "http://fakeurl.com", + 0.25, + [ + {"key_1": "value_1"}, + {"key_2": "value_2"}]) + exception_raised = str(exc_info.value) + assert "401" in exception_raised + assert "Failed to POST http://fakeurl.com" in exception_raised + + + def test_send_batch_not_found(self): + """Should raise an exception if the request is not found.""" + iot = IoT() + with patch('requests.post') as mock_post: + mock_post.return_value.status_code = 404 + with pytest.raises(exceptions.FetchError) as exc_info: + iot.send_batch("fake_sensor_name", + "fake_api_key", + "http://fakeurl.com", + 0.25, + [ + {"key_1": "value_1"}, + {"key_2": "value_2"}]) + + exception_raised = str(exc_info.value) + assert "404" in exception_raised + assert "Failed to POST http://fakeurl.com" in exception_raised + + + def test_send_batch_server_error(self): + """Should raise an exception if there is a server error.""" + iot = IoT() + with patch('requests.post') as mock_post: + mock_post.return_value.status_code = 500 + with pytest.raises(exceptions.FetchError) as exc_info: + iot.send_batch("fake_sensor_name", + "fake_api_key", + "http://fakeurl.com", + 0.25, + [ + {"key_1": "value_1"}, + {"key_2": "value_2"}]) + + exception_raised = str(exc_info.value) + assert "500" in exception_raised + assert "Failed to POST http://fakeurl.com" in exception_raised \ No newline at end of file