diff --git a/.github/workflows/python-prereleased.yml b/.github/workflows/python-prereleased.yml index 28615bd..28b6b87 100644 --- a/.github/workflows/python-prereleased.yml +++ b/.github/workflows/python-prereleased.yml @@ -25,7 +25,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v3 with: - python-version: "3.9" + python-version: "3.12" - name: Install dependencies run: | python -m pip install --upgrade pip diff --git a/.github/workflows/python-pull-request.yml b/.github/workflows/python-pull-request.yml index 9a7000e..25c1d89 100644 --- a/.github/workflows/python-pull-request.yml +++ b/.github/workflows/python-pull-request.yml @@ -27,7 +27,7 @@ jobs: strategy: max-parallel: 1 matrix: - version: ["3.8", "3.9"] + version: [ "3.12", "3.11", "3.10", "3.9" ] runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -61,4 +61,4 @@ jobs: export DEVO_SENDER_CHAIN=$(realpath certs/us/ca.crt) export TMPDIR=${PWD} cd tests - python -m pytest + python -m pytest -vvv diff --git a/.github/workflows/python-released.yml b/.github/workflows/python-released.yml index bf37d56..f394b57 100644 --- a/.github/workflows/python-released.yml +++ b/.github/workflows/python-released.yml @@ -25,7 +25,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v3 with: - python-version: "3.9" + python-version: "3.12" - name: Install dependencies run: | python -m pip install --upgrade pip diff --git a/CHANGELOG.md b/CHANGELOG.md index 8e151b9..da1e9fd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,33 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). +## [6.0.0] - 2024-10-07 + +### Changed + - Supported Python versions extended to 10, 11 and 12 + - Added time zones in date operations + - Jobs API reviewed and fixed. Jobs searching by type and friendlyName discontinued as it is not supported by API. + Jobs API unit test checked and enabled + - Added timeout to unit tests of API queries. They may run forever when faulty + +### Fixed + - Keep-alive mechanism not working for queries with `destination`. Forcing NO_KEEP_ALIVE in queries with + `destination`. + - SSL wrapping of the TCP connection when no certificates are used improved + - Fix auxiliary Echo serving for unit testing in order to run with new async paradigm + - Documentation fixes. Some parameters missing or non-existent in docstring + - Fix for a unit test when using concurrency (from library `stopit` to `pebble`) + +### Removed + - Python 3.8 support discontinued + +### Incompatibilities with 5.x.x that caused mayor version bump + - Python 3.8 not supported anymore + - Jobs searching by type and friendlyName discontinued in Jobs API. Only search by job id is supported. + - Date requires time zone + - Query with `destination` are forced to NO_KEEP_ALIVE mode for Keep-alive mechanism (instead of + DEFAULT_KEEPALIVE_TOKEN) + ## [5.4.1] - 2024-09-13 ### Security diff --git a/README.md b/README.md index d5716f6..ea8409b 100755 --- a/README.md +++ b/README.md @@ -14,11 +14,11 @@ This is the SDK to access Devo directly from Python. It can be used to: ## Requirements -The Devo SDK for Python requires Python 3.8+ +The Devo SDK for Python requires Python 3.9+ ## Compatibility -- Tested compatibility for python 3.8 and 3.9 +- Tested compatibility for python 3.9, 3.10, 3.11 and 3.12 ## Quick Start diff --git a/devo/__version__.py b/devo/__version__.py index 629774f..2d099da 100644 --- a/devo/__version__.py +++ b/devo/__version__.py @@ -1,6 +1,6 @@ __description__ = "Devo Python Library." __url__ = "http://www.devo.com" -__version__ = "5.4.1" +__version__ = "6.0.0" __author__ = "Devo" __author_email__ = "support@devo.com" __license__ = "MIT" diff --git a/devo/api/client.py b/devo/api/client.py index b2bff14..3645c18 100644 --- a/devo/api/client.py +++ b/devo/api/client.py @@ -55,6 +55,8 @@ "connection_error": "Failed to establish a new connection", "other_errors": "Error while invoking query", "error_no_detail": "Error code %d while invoking query", + "no_keepalive_for_destination": "Queries with destination functionality only support No Keepalive mode. Forced to" + " NO_KEEP_ALIVE" } DEFAULT_KEEPALIVE_TOKEN = "\n" @@ -173,6 +175,7 @@ def __init__( self.processor = None self.set_processor(processor) self.keepAliveToken = None + self.set_keepalive_token(keepAliveToken) if pragmas: @@ -235,7 +238,12 @@ def set_keepalive_token(self, keepAliveToken=DEFAULT_KEEPALIVE_TOKEN): # keepalive (cannot be modified), but implementation uses # NO_KEEP_ALIVE value as it does not change the query msgpack and # xls does not support keepalive - if self.response in [ + # Queries with destination only supports NO_KEEP_ALIVE + if self.destination is not None: + self.keepAliveToken = NO_KEEPALIVE_TOKEN + if keepAliveToken not in [NO_KEEPALIVE_TOKEN, DEFAULT_KEEPALIVE_TOKEN]: + logging.warning(ERROR_MSGS["no_keepalive_for_destination"]) + elif self.response in [ "json", "json/compact", "json/simple", @@ -254,7 +262,6 @@ def set_keepalive_token(self, keepAliveToken=DEFAULT_KEEPALIVE_TOKEN): self.keepAliveToken = NO_KEEPALIVE_TOKEN return True - class Client: """ The Devo search rest api main class @@ -441,6 +448,7 @@ def query( :param limit: Max number of rows :param offset: start of needle for query :param comment: comment for query + :param ip_as_string: whether to recive IP types as strings :return: Result of the query (dict) or Iterator object """ dates = self._generate_dates(dates) @@ -755,18 +763,10 @@ def _generate_pragmas(self, comment=None): return str_pragmas - def get_jobs(self, job_type=None, name=None): + def get_jobs(self): """Get list of jobs by type and name, default All - :param job_type: category of jobs - :param name: name of jobs :return: json""" - plus = ( - "" - if not job_type - else "/{}".format(job_type if not name else "{}/{}".format(job_type, name)) - ) - - return self._call_jobs("{}{}{}".format(self.address[0], "/search/jobs", plus)) + return self._call_jobs("{}{}".format(self.address[0], "/search/jobs")) def get_job(self, job_id): """Get all info of job diff --git a/devo/api/scripts/client_cli.py b/devo/api/scripts/client_cli.py index afb5ee7..6d8e5d7 100644 --- a/devo/api/scripts/client_cli.py +++ b/devo/api/scripts/client_cli.py @@ -161,7 +161,7 @@ def configure(args): """ Load CLI configuration :param args: args from files, launch vars, etc - :return: Clien t API Object and Config values in array + :return: Client API Object and Config values in array """ config = Configuration() try: diff --git a/devo/common/dates/dateoperations.py b/devo/common/dates/dateoperations.py index 8e955af..eb90f8e 100644 --- a/devo/common/dates/dateoperations.py +++ b/devo/common/dates/dateoperations.py @@ -2,6 +2,9 @@ """A collection of allowed operations on date parsing""" from datetime import datetime as dt +import zoneinfo +UTC = zoneinfo.ZoneInfo("UTC") + from datetime import timedelta from .dateutils import to_millis, trunc_time, trunc_time_minute @@ -60,7 +63,7 @@ def now(): Return current millis in UTC :return: Millis """ - return to_millis(dt.utcnow()) + return to_millis(dt.now(UTC)) def now_without_ms(): @@ -68,7 +71,7 @@ def now_without_ms(): Return current millis in UTC :return: Millis """ - return to_millis(trunc_time_minute(dt.utcnow())) + return to_millis(trunc_time_minute(dt.now(UTC))) def today(): @@ -76,7 +79,7 @@ def today(): Return current millis with the time truncated to 00:00:00 :return: Millis """ - return to_millis(trunc_time(dt.utcnow())) + return to_millis(trunc_time(dt.now(UTC))) def yesterday(): @@ -84,7 +87,7 @@ def yesterday(): Return millis from yesterday with time truncated to 00:00:00 :return: Millis """ - return to_millis(trunc_time(dt.utcnow()) - timedelta(days=1)) + return to_millis(trunc_time(dt.now(UTC)) - timedelta(days=1)) def parse_functions(): diff --git a/devo/common/dates/dateutils.py b/devo/common/dates/dateutils.py index bca1aa1..84699c9 100644 --- a/devo/common/dates/dateutils.py +++ b/devo/common/dates/dateutils.py @@ -2,6 +2,8 @@ """Utils for format and trunc dates.""" from datetime import datetime as dt +import zoneinfo +UTC = zoneinfo.ZoneInfo("UTC") def to_millis(date): @@ -10,7 +12,10 @@ def to_millis(date): :param date: Date for parse to millis :return: Millis from the date """ - return int((date - dt.utcfromtimestamp(0)).total_seconds() * 1000) + # Verify whether param has timezone, if not set the default UTC one + if date.tzinfo is None: + date = date.replace(tzinfo=UTC) + return int((date - dt.fromtimestamp(0, UTC)).total_seconds() * 1000) def trunc_time(date): @@ -50,4 +55,4 @@ def get_timestamp(): Generate current timestamp :return: """ - return to_millis(dt.utcnow()) + return to_millis(dt.now(UTC)) diff --git a/devo/sender/data.py b/devo/sender/data.py index f72ae80..b831430 100644 --- a/devo/sender/data.py +++ b/devo/sender/data.py @@ -15,6 +15,7 @@ from ssl import SSLWantReadError, SSLWantWriteError from threading import Thread, Lock, Event from typing import Optional, Callable +import warnings import pem from _socket import SHUT_WR @@ -74,6 +75,7 @@ def __str__(self): RAW_SENDING_ERROR = ("Error sending raw event: %s",) CLOSING_ERROR = "Error closing connection" FLUSHING_BUFFER_ERROR = "Error flushing buffer" + ERROR_AFTER_TIMEOUT = "Timeout reached" class DevoSenderException(Exception): @@ -593,8 +595,6 @@ def __connect_ssl(self): context = ssl.create_default_context(cafile=self._sender_config.chain) context.options |= ssl.OP_NO_SSLv2 context.options |= ssl.OP_NO_SSLv3 - context.options |= ssl.OP_NO_TLSv1 - context.options |= ssl.OP_NO_TLSv1_1 context.minimum_version = ssl.TLSVersion.TLSv1_2 context.maximum_version = ssl.TLSVersion.TLSv1_3 @@ -614,9 +614,16 @@ def __connect_ssl(self): self.socket, server_hostname=self._sender_config.address[0] ) else: - self.socket = ssl.wrap_socket( - self.socket, ssl_version=ssl.PROTOCOL_TLS, cert_reqs=ssl.CERT_NONE - ) + context = ssl.create_default_context(ssl.Purpose.SERVER_AUTH) + context.check_hostname = False + context.verify_mode = ssl.CERT_NONE + context.options |= ssl.OP_NO_SSLv2 + context.options |= ssl.OP_NO_SSLv3 + context.minimum_version = ssl.TLSVersion.TLSv1_2 + context.maximum_version = ssl.TLSVersion.TLSv1_3 + self.logger.warning("One or more of CA certificate, private or public certificate is not provided" + " and TLS unsecure connection is established") + self.socket = context.wrap_socket(self.socket) self.socket.connect(self._sender_config.address) self.last_message = int(time.time()) @@ -643,7 +650,7 @@ def info(self, msg): """ self.send(tag=self.logging.get("tag"), msg=msg) - # TODO: Deprecated + # TODO: Deprecated, to remove in next mayor version def set_sec_level(self, sec_level=None): """ Set sec_level of SSL Context: @@ -651,9 +658,11 @@ def set_sec_level(self, sec_level=None): :param sec_level: sec_level value :return """ + warnings.warn("This function is deprecated and it will be removed in future versions", + DeprecationWarning, stacklevel=2) self._sender_config.sec_level = sec_level - # TODO: Deprecated + # TODO: Deprecated, to remove in next mayor version def set_verify_mode(self, verify_mode=None): """ Set verify_mode of SSL Context: @@ -665,9 +674,11 @@ def set_verify_mode(self, verify_mode=None): :param verify_mode: verify mode value :return """ + warnings.warn("This function is deprecated and it will be removed in future versions", + DeprecationWarning, stacklevel=2) self._sender_config.verify_mode = verify_mode - # TODO: Deprecated + # TODO: Deprecated, to remove in next mayor version def set_check_hostname(self, check_hostname=True): """ Set check_hostname of SSL Context: @@ -675,6 +686,8 @@ def set_check_hostname(self, check_hostname=True): :param check_hostname: check_hostname value. Default True :return """ + warnings.warn("This function is deprecated and it will be removed in future versions", + DeprecationWarning, stacklevel=2) self._sender_config.check_hostname = check_hostname def buffer_size(self, size=19500): @@ -1093,7 +1106,6 @@ def for_logging(config=None, con_type=None, tag=None, level=None): :param con_type: type of connection :param tag: tag for the table :param level: level of logger - :param formatter: log formatter :return: Sender object """ con = Sender(config=config, con_type=con_type) diff --git a/devo/sender/lookup.py b/devo/sender/lookup.py index 0813169..3c752ed 100644 --- a/devo/sender/lookup.py +++ b/devo/sender/lookup.py @@ -392,6 +392,7 @@ def field_to_str(field, escape_quotes=False): """ Convert one value to STR, cleaning it :param field: field to clean + :param escape_quotes: whether to escape quotes in response :return: """ return ",%s" % Lookup.clean_field(field, escape_quotes) @@ -402,6 +403,7 @@ def process_fields(fields=None, key_index=None, escape_quotes=False): Method to convert list with one row/fields to STR to send :param fields: fields list :param key_index: index of key in fields + :param escape_quotes: whether to escape quotes in response :return: """ # First the key @@ -417,6 +419,7 @@ def clean_field(field=None, escape_quotes=False): """ Strip and quotechar the fields :param str field: field for clean + :param escape_quotes: whether to escape quotes in response :return str: cleaned field """ if not isinstance(field, (str, bytes)): diff --git a/docs/api/api.md b/docs/api/api.md index eda9942..0239db8 100644 --- a/docs/api/api.md +++ b/docs/api/api.md @@ -496,6 +496,7 @@ Client support several modes for supporting this mechanism. The mode is set up i * `json`, `json/compact`, `json/simple` and `json/simple/compact` token is always `b' '` (four utf-8 spaces chars) * For `csv` and `tsv` token is the custom `str` set as parameter * `msgpack` and `xls` do not support this mode +* Queries using `destination` functionality does not support keep alive. NO_KEEP_ALIVE is forced | Response mode | default mode | `NO_KEEPALIVE_TOKEN` | `DEFAULT_KEEPALIVE_TOKEN` | `EMPTY_EVENT_KEEPALIVE_TOKEN` | Custom keep alive token | |---------------------|---------------------------|----------------------|---------------------------|-------------------------------|------------------------------| diff --git a/requirements-test.txt b/requirements-test.txt index 698b033..0e50fe7 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,7 +1,8 @@ -stopit==1.1.2 msgpack~=1.0.8 responses~=0.25.3 pipdeptree~=2.23.0 pytest~=8.2.2 pytest-cov~=5.0.0 -mock==5.1.0 \ No newline at end of file +mock==5.1.0 +pebble==5.0.7 +pytest-timeout~=2.3.1 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index a399d87..cb46719 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ click==8.1.7 -PyYAML==6.0.1 +PyYAML~=6.0.1 requests~=2.32 pem~=21.2.0 pyopenssl~=24.2.1 diff --git a/setup.py b/setup.py index e67b4ff..f949afc 100644 --- a/setup.py +++ b/setup.py @@ -18,8 +18,10 @@ "Operating System :: OS Independent", "Programming Language :: Python", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", "Topic :: Software Development :: Libraries :: Python Modules", @@ -27,7 +29,7 @@ INSTALL_REQUIRES = [ "requests~=2.32", "click==8.1.7", - "PyYAML==6.0.1", + "PyYAML~=6.0.1", "pem~=21.2.0", "pyopenssl~=24.2.1", "pytz~=2024.1", @@ -36,12 +38,13 @@ ] EXTRAS_REQUIRE = { "dev": [ - "stopit==1.1.2", "msgpack~=1.0.8", "responses~=0.25.3", "pipdeptree~=2.23.0", "pytest~=8.2.2", "pytest-cov~=5.0.0", + "mock~=5.1.0", + "pebble~=5.0.7" ] } CLI = [ diff --git a/tests/integration/local_servers.py b/tests/integration/local_servers.py index f70cf01..665beb7 100644 --- a/tests/integration/local_servers.py +++ b/tests/integration/local_servers.py @@ -32,44 +32,57 @@ def wait_for_ready_server(address, port): except socket.error: num_tries -= 1 time.sleep(1) + if num_tries == 0: + raise Exception("Connection to address %s at port %d could not be established" % (address, port)) -class SSLServer: +class EchoServer: - def __init__(self, ip="127.0.0.1", port=4488, certfile=None, keyfile=None): + def __init__(self, ip="127.0.0.1", port=4488, certfile=None, keyfile=None, ssl=True): self.ip = ip self.port = port self.cert = certfile self.key = keyfile self.shutdown = False + self.ssl = ssl self.file_path = "".join((os.path.dirname(os.path.abspath(__file__)), os.sep)) - self.server_process = multiprocessing.Process(target=self.server, name="sslserver") + self.server_process = multiprocessing.Process(target=self.server, name="sslserver" if ssl else "tcpserver") self.server_process.start() def server(self): + asyncio.run(self.run_server()) - @asyncio.coroutine - def handle_connection(reader, writer): + async def run_server(self): + + async def handle_connection(reader, writer): addr = writer.get_extra_info("peername") try: - while True: - data = yield from reader.read(500) - print("Server received {!r} from {}".format(data, addr)) + while not self.shutdown: + data = await reader.read(500) assert len(data) > 0, repr(data) + print("Server received {!r} from {}".format(data, addr)) writer.write(data) - yield from writer.drain() - except Exception: + await writer.drain() + except Exception as e: + print(f"Error: {e}") + finally: writer.close() + await writer.wait_closed() + + # Create SSL context + sc = None + if self.ssl: + sc = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH) + sc.load_cert_chain(self.cert, self.key) - sc = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH) - sc.load_cert_chain(self.cert, self.key) + # Run server + server = await asyncio.start_server(handle_connection, self.ip, self.port, ssl=sc) - loop = asyncio.get_event_loop() - coro = asyncio.start_server(handle_connection, self.ip, self.port, ssl=sc, loop=loop) - server = loop.run_until_complete(coro) + print("Serving SSL on {}".format(server.sockets[0].getsockname())) - print("Serving on {}".format(server.sockets[0].getsockname())) - loop.run_forever() + async with server: + # Server runs forever (until closed) + await server.serve_forever() def close_server(self): self.shutdown = True @@ -77,31 +90,5 @@ def close_server(self): self.server_process.join() -class TCPServer: - - def __init__(self, ip="127.0.0.1", port=4489): - self.ip = ip - self.port = port - self.shutdown = False - self.server = None - self.file_path = "".join((os.path.dirname(os.path.abspath(__file__)), os.sep)) - self.server = threading.Thread(target=self.start_server, kwargs={"ip": ip, "port": port}) - self.server.setDaemon(True) - self.server.start() - - def start_server(self): - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - s.bind((self.ip, self.port)) - s.listen(5) - conn, addr = s.accept() - while not self.shutdown: - data = conn.recv(8000) - conn.send(data) - - def close_server(self): - self.shutdown = True - - if __name__ == "__main__": print("Trying to run module local_servers.py directly...") diff --git a/tests/integration/test_api_cli.py b/tests/integration/test_api_cli.py index 1d06227..b2ac2b5 100644 --- a/tests/integration/test_api_cli.py +++ b/tests/integration/test_api_cli.py @@ -204,6 +204,7 @@ def test_bad_credentials(api_config): assert result.exception.code == 12 +@pytest.mark.timeout(180) def test_normal_query(api_config): runner = CliRunner() result = runner.invoke( @@ -228,6 +229,7 @@ def test_normal_query(api_config): assert '{"m":{"eventdate":{"type":"timestamp","index":0' in result.output +@pytest.mark.timeout(180) def test_with_config_file(api_config): if api_config.config_path: runner = CliRunner() @@ -249,6 +251,7 @@ def test_with_config_file(api_config): assert '{"m":{"eventdate":{"type":"timestamp","index":0' in result.output +@pytest.mark.timeout(180) def test_query_with_ip_as_int(api_config): runner = CliRunner() result = runner.invoke( @@ -278,6 +281,7 @@ def test_query_with_ip_as_int(api_config): assert isinstance(resp_data["d"][0], int) +@pytest.mark.timeout(180) def test_query_with_ip_as_str(api_config): runner = CliRunner() result = runner.invoke( diff --git a/tests/integration/test_api_query.py b/tests/integration/test_api_query.py index fd229d8..5aa86c4 100644 --- a/tests/integration/test_api_query.py +++ b/tests/integration/test_api_query.py @@ -3,18 +3,19 @@ import tempfile import types from datetime import datetime, timedelta +import zoneinfo from ssl import CERT_NONE -from time import gmtime, strftime - import pytest -import stopit from ip_validation import is_valid_ip - +from pebble import concurrent +from concurrent.futures import TimeoutError from devo.api import Client, ClientConfig, DevoClientException from devo.common import Configuration from devo.common.loadenv.load_env import load_env_file from devo.sender.data import Sender, SenderConfigSSL +UTC = zoneinfo.ZoneInfo("UTC") + # Load environment variables form test directory load_env_file(os.path.abspath(os.getcwd()) + os.sep + "environment.env") @@ -150,6 +151,7 @@ def test_from_dict(api_config): assert isinstance(api, Client) +@pytest.mark.timeout(180) def test_simple_query(api_config): config = ClientConfig(stream=False, response="json") @@ -165,6 +167,7 @@ def test_simple_query(api_config): assert len(json.loads(result)["object"]) > 0 +@pytest.mark.timeout(180) def test_token(api_config): api = Client( auth={"token": api_config.api_token}, @@ -177,6 +180,7 @@ def test_token(api_config): assert len(json.loads(result)["object"]) > 0 +@pytest.mark.timeout(180) def test_query_id(api_config): api = Client( auth={"key": api_config.api_key, "secret": api_config.api_secret}, @@ -190,6 +194,7 @@ def test_query_id(api_config): assert isinstance(len(json.loads(result)["object"]), int) +@pytest.mark.timeout(180) def test_query_yesterday_to_today(api_config): api = Client( auth={"key": api_config.api_key, "secret": api_config.api_secret}, @@ -204,6 +209,7 @@ def test_query_yesterday_to_today(api_config): assert len(json.loads(result)["object"]) == 1 +@pytest.mark.timeout(180) def test_query_from_seven_days(api_config): api = Client( auth={"key": api_config.api_key, "secret": api_config.api_secret}, @@ -216,6 +222,7 @@ def test_query_from_seven_days(api_config): assert len(json.loads(result)["object"]) == 1 +@pytest.mark.timeout(180) def test_query_from_fixed_dates(api_config): api = Client( auth={"key": api_config.api_key, "secret": api_config.api_secret}, @@ -226,14 +233,15 @@ def test_query_from_fixed_dates(api_config): result = api.query( query=api_config.query, dates={ - "from": strftime("%Y-%m-%d", gmtime()), - "to": strftime("%Y-%m-%d %H:%M:%S", gmtime()), + "from": datetime.now(UTC).strftime("%Y-%m-%d"), + "to": datetime.now(UTC).strftime("%Y-%m-%d %H:%M:%S"), }, ) assert result is not None assert len(json.loads(result)["object"]) == 1 +@pytest.mark.timeout(180) def test_stream_query(api_config): api = Client( auth={"key": api_config.api_key, "secret": api_config.api_secret}, @@ -247,6 +255,7 @@ def test_stream_query(api_config): assert len(result) == 1 +@pytest.mark.timeout(180) def test_stream_query_no_results_bounded_dates(api_config): api = Client( auth={"key": api_config.api_key, "secret": api_config.api_secret}, @@ -270,17 +279,21 @@ def test_stream_query_no_results_unbounded_dates(api_config): result = api.query(query=api_config.query_no_results) assert isinstance(result, types.GeneratorType) + @concurrent.process(timeout=3) + def fetch_result(result): + return list(result) + + future = fetch_result(result) + try: - with stopit.ThreadingTimeout(3) as to_ctx_mgr: - result = list(result) - except DevoClientException: - # This exception is sent because - # devo.api.client.Client._make_request catches the - # stopit.TimeoutException, but the latter is not - # wrapped, so we cannot obtain it from here. - assert to_ctx_mgr.state == to_ctx_mgr.TIMED_OUT + results = future.result() + except TimeoutError: + assert True + except DevoClientException as error: + assert False, "DevoClientException raised: %s" % (error) +@pytest.mark.timeout(180) def test_pragmas(api_config): """Test the api when the pragma comment.free is used""" api = Client( @@ -296,6 +309,7 @@ def test_pragmas(api_config): assert len(json.loads(result)["object"]) == 1 +@pytest.mark.timeout(180) def test_pragmas_not_comment_free(api_config): """Test the api when the pragma comment.free is not used""" api = Client( @@ -312,6 +326,7 @@ def test_pragmas_not_comment_free(api_config): @pytest.mark.skip(reason="This is an internal functionality, not intended for external use") +@pytest.mark.timeout(180) def test_unsecure_http_query(api_config): """ This test is intended for checking unsecure HTTP requests. Devo will @@ -432,6 +447,7 @@ def test_msgpack_future_queries(api_config): ) +@pytest.mark.timeout(180) def test_query_with_ip_as_int(api_config): config = ClientConfig(stream=False, response="json") @@ -450,6 +466,7 @@ def test_query_with_ip_as_int(api_config): assert isinstance(resp_data[api_config.field_with_ip], int) +@pytest.mark.timeout(180) def test_query_with_ip_as_string(api_config): config = ClientConfig(stream=False, response="json") diff --git a/tests/integration/test_api_tasks.py b/tests/integration/test_api_tasks.py index 3159938..5dfbe7f 100644 --- a/tests/integration/test_api_tasks.py +++ b/tests/integration/test_api_tasks.py @@ -1,5 +1,6 @@ +import json import os - +import uuid import pytest from devo.api import Client @@ -10,7 +11,7 @@ @pytest.fixture(scope="module") -def setup_client(): +def setup_client(job_name): yield Client( config={ "key": os.getenv("DEVO_API_KEY", None), @@ -19,7 +20,7 @@ def setup_client(): "stream": False, "destination": { "type": "donothing", - "params": {"friendlyName": "devo-sdk-api-test"}, + "params": {"friendlyName": job_name}, }, } ) @@ -30,38 +31,42 @@ def setup_query(): yield "from siem.logtrust.web.connection select action" -@pytest.mark.skip("temporarily disabled due to Query API bug") -def test_jobs_cycle(setup_client, setup_query): - setup_client.query(query=setup_query, dates={"from": "1d"}) +@pytest.fixture(scope="module") +def job_name(): + return "devo-sdk-api-test" + str(uuid.uuid1().int) - # Get all jobs - result = setup_client.get_jobs() - assert result["object"] is True - # Get job by name - result = setup_client.get_jobs(name="devo-sdk-api-test") - assert result["object"] is True +def test_jobs_cycle(setup_client, setup_query, job_name): + result = setup_client.query(query=setup_query, dates={"from": "1d", "to": "endday"}) + result = json.loads(result) + assert result["status"] == 0 + assert "object" in result + assert "id" in result["object"] + job_id = result["object"]["id"] - # Get job by type - result = setup_client.get_jobs(job_type="donothing") - assert result["object"] is True + # Get all jobs + result = setup_client.get_jobs() + assert len(result["object"]) > 0 - # Get job by name and type - result = setup_client.get_jobs(name="devo-sdk-api-test", job_type="donothing") - assert result["object"] is True - job_id = result["object"][0]["id"] + # Get job by job id + result = setup_client.get_job(job_id=job_id) + assert result["object"]["friendlyName"] == job_name + assert result["object"]["id"] == job_id # Stop job by id result = setup_client.stop_job(job_id) - assert result["object"]["status"] == "STOPPED" + assert result["object"]["status"] in ["STOPPED", "COMPLETED"] # Start job by id result = setup_client.start_job(job_id) - assert result["object"]["status"] == "RUNNING" + assert result["object"]["status"] in ["RUNNING", "COMPLETED"] - # Delete job by id - result = setup_client.remove_job(job_id) - assert result["object"]["status"] == "REMOVED" + # Delete all jobs created by this and past test execution (cleaning purposes) + result = setup_client.get_jobs() + jobs_ids = [job["id"] for job in result["object"] if job["friendlyName"].startswith("devo-sdk-api-test")] + for job_id in jobs_ids: + result = setup_client.remove_job(job_id) + assert result["object"]["status"] in ["REMOVED", "COMPLETED"] if __name__ == "__main__": diff --git a/tests/integration/test_sender_cli.py b/tests/integration/test_sender_cli.py index 5900716..b292ce5 100644 --- a/tests/integration/test_sender_cli.py +++ b/tests/integration/test_sender_cli.py @@ -4,7 +4,7 @@ import pytest from click.testing import CliRunner -from local_servers import SSLServer, find_available_port, wait_for_ready_server +from local_servers import EchoServer, find_available_port, wait_for_ready_server from devo.common import Configuration from devo.common.generic.configuration import ConfigurationException @@ -102,11 +102,12 @@ class Fixture: setup.bad_yaml_config_path = setup.common_path + os.sep + "bad_yaml_config.yaml" setup.ssl_port = find_available_port(setup.ssl_address, setup.ssl_port) - local_ssl_server = SSLServer( + local_ssl_server = EchoServer( setup.ssl_address, setup.ssl_port, setup.local_server_cert, setup.local_server_key, + ssl=True ) wait_for_ready_server(local_ssl_server.ip, local_ssl_server.port) diff --git a/tests/integration/test_sender_send_data.py b/tests/integration/test_sender_send_data.py index dfefb43..83aa454 100644 --- a/tests/integration/test_sender_send_data.py +++ b/tests/integration/test_sender_send_data.py @@ -8,7 +8,7 @@ import pem import pytest -from local_servers import (SSLServer, TCPServer, find_available_port, +from local_servers import (EchoServer, find_available_port, wait_for_ready_server) from OpenSSL import SSL, crypto @@ -66,7 +66,7 @@ class Fixture: setup.local_server_chain = os.getenv( "DEVO_SENDER_SERVER_CHAIN", f"{setup.certs_path}/ca/ca_cert.pem" ) - setup.test_tcp = os.getenv("DEVO_TEST_TCP", False) + setup.test_tcp = (os.getenv("DEVO_TEST_TCP", False) == 'True') setup.configuration = Configuration() setup.configuration.set( "sender", @@ -101,14 +101,14 @@ class Fixture: # Run local servers # ---------------------------------------- setup.ssl_port = find_available_port(setup.ssl_address, setup.ssl_port) - local_ssl_server = SSLServer( - setup.ssl_address, setup.ssl_port, setup.local_server_cert, setup.local_server_key + local_ssl_server = EchoServer( + setup.ssl_address, setup.ssl_port, setup.local_server_cert, setup.local_server_key, ssl=True ) wait_for_ready_server(local_ssl_server.ip, local_ssl_server.port) if setup.test_tcp: setup.tcp_port = find_available_port(setup.tcp_address, setup.tcp_port) - local_tcp_server = TCPServer(setup.tcp_address, setup.tcp_port) + local_tcp_server = EchoServer(setup.tcp_address, setup.tcp_port, ssl=False) wait_for_ready_server(local_tcp_server.ip, local_tcp_server.port) yield setup @@ -262,19 +262,15 @@ def test_rt_send_no_certs(setup): """ if not setup.test_tcp: pytest.skip("Not testing TCP") - try: - engine_config = SenderConfigSSL( - address=(setup.ssl_address, setup.ssl_port), - check_hostname=False, - verify_mode=CERT_NONE, - ) - con = Sender(engine_config) - for i in range(setup.default_numbers_sendings): - con.send(tag=setup.my_app, msg=setup.test_msg) - con.close() - return True - except Exception: - return False + engine_config = SenderConfigSSL( + address=(setup.ssl_address, setup.ssl_port), + check_hostname=False, + verify_mode=CERT_NONE, + ) + con = Sender(engine_config) + for i in range(setup.default_numbers_sendings): + con.send(tag=setup.my_app, msg=setup.test_msg) + con.close() def test_sender_as_handler(setup): diff --git a/tests/integration/test_sender_send_lookup.py b/tests/integration/test_sender_send_lookup.py index 339405f..cda9f27 100644 --- a/tests/integration/test_sender_send_lookup.py +++ b/tests/integration/test_sender_send_lookup.py @@ -6,7 +6,7 @@ from unittest import mock import pytest -from local_servers import SSLServer, find_available_port, wait_for_ready_server +from local_servers import EchoServer, find_available_port, wait_for_ready_server from devo.common import Configuration from devo.common.loadenv.load_env import load_env_file @@ -97,11 +97,12 @@ class Fixture: setup.configuration.save(path=setup.config_path) setup.ssl_port = find_available_port(setup.ssl_address, setup.ssl_port) - local_ssl_server = SSLServer( + local_ssl_server = EchoServer( setup.ssl_address, setup.ssl_port, setup.local_server_cert, setup.local_server_key, + ssl=True ) wait_for_ready_server(local_ssl_server.ip, local_ssl_server.port) diff --git a/tests/unit/test_common_date_parser.py b/tests/unit/test_common_date_parser.py index 8053036..92219b7 100644 --- a/tests/unit/test_common_date_parser.py +++ b/tests/unit/test_common_date_parser.py @@ -1,4 +1,6 @@ from datetime import datetime as dt +import zoneinfo +UTC = zoneinfo.ZoneInfo("UTC") import pytest @@ -7,7 +9,7 @@ @pytest.fixture(scope="module") def setup_epoch(): - epoch = dt.utcfromtimestamp(0) + epoch = dt.fromtimestamp(0, UTC) yield epoch @@ -15,13 +17,13 @@ def setup_epoch(): # -------------------------------------------------------------------------- def test_default_to(setup_epoch): ts1 = str(default_to())[:11] - ts2 = str((dt.utcnow() - setup_epoch).total_seconds() * 1000)[:11] + ts2 = str((dt.now(UTC) - setup_epoch).total_seconds() * 1000)[:11] assert ts1 == ts2 def test_default_from(setup_epoch): ts1 = str(default_from())[:11] - ts2 = str(int((dt.utcnow() - setup_epoch).total_seconds() * 1000) - 86400000)[:11] + ts2 = str(int((dt.now(UTC) - setup_epoch).total_seconds() * 1000) - 86400000)[:11] assert ts1 == ts2 @@ -54,16 +56,17 @@ def test_month(): # Tests relatives # -------------------------------------------------------------------------- def test_now(setup_epoch): - assert default_from("now()") == int((dt.utcnow() - setup_epoch).total_seconds() * 1000) + tolerance = 100 + assert abs(default_from("now()") - int((dt.now(UTC) - setup_epoch).total_seconds() * 1000)) < 100 def test_today(setup_epoch): - tmp = dt.utcnow().replace(hour=0, minute=0, second=0, microsecond=0) + tmp = dt.now(UTC).replace(hour=0, minute=0, second=0, microsecond=0) assert default_from("today()") == int((tmp - setup_epoch).total_seconds() * 1000) def test_yesterday(setup_epoch): - tmp = dt.utcnow().replace(hour=0, minute=0, second=0, microsecond=0) + tmp = dt.now(UTC).replace(hour=0, minute=0, second=0, microsecond=0) assert ( default_from("yesterday()") == int((tmp - setup_epoch).total_seconds() * 1000) - 86400000 )