Merge pull request #292 from DevoInc/python12support

Version 6.0.0 merge request

.github/workflows/python-prereleased.yml

@@ -25,7 +25,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v3
         with:
-          python-version: "3.9"
+          python-version: "3.12"
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip

.github/workflows/python-pull-request.yml

@@ -27,7 +27,7 @@ jobs:
     strategy:
       max-parallel: 1
       matrix:
-        version: ["3.8", "3.9"]
+        version: [ "3.12", "3.11", "3.10", "3.9" ]
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
@@ -61,4 +61,4 @@ jobs:
           export DEVO_SENDER_CHAIN=$(realpath certs/us/ca.crt)
           export TMPDIR=${PWD}
           cd tests
-          python -m pytest
+          python -m pytest -vvv

.github/workflows/python-released.yml

@@ -25,7 +25,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v3
         with:
-          python-version: "3.9"
+          python-version: "3.12"
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip

CHANGELOG.md

@@ -4,6 +4,33 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## [6.0.0] - 2024-10-07
+
+### Changed
+ - Supported Python versions extended to 10, 11 and 12
+ - Added time zones in date operations
+ - Jobs API reviewed and fixed. Jobs searching by type and friendlyName discontinued as it is not supported by API.
+   Jobs API unit test checked and enabled
+ - Added timeout to unit tests of API queries. They may run forever when faulty
+
+### Fixed
+ - Keep-alive mechanism not working for queries with `destination`. Forcing NO_KEEP_ALIVE in queries with
+   `destination`. 
+ - SSL wrapping of the TCP connection when no certificates are used improved
+ - Fix auxiliary Echo serving for unit testing in order to run with new async paradigm
+ - Documentation fixes. Some parameters missing or non-existent in docstring
+ - Fix for a unit test when using concurrency (from library `stopit` to `pebble`)
+
+### Removed
+ - Python 3.8 support discontinued
+
+### Incompatibilities with 5.x.x that caused mayor version bump
+ - Python 3.8 not supported anymore
+ - Jobs searching by type and friendlyName discontinued in Jobs API. Only search by job id is supported.
+ - Date requires time zone
+ - Query with `destination` are forced to NO_KEEP_ALIVE mode for Keep-alive mechanism (instead of
+   DEFAULT_KEEPALIVE_TOKEN)
+
 ## [5.4.1] - 2024-09-13
 
 ### Security

README.md

@@ -14,11 +14,11 @@ This is the SDK to access Devo directly from Python. It can be used to:
 
 ## Requirements
 
-The Devo SDK for Python requires Python 3.8+
+The Devo SDK for Python requires Python 3.9+
 
 ## Compatibility
 
-- Tested compatibility for python 3.8 and 3.9
+- Tested compatibility for python 3.9, 3.10, 3.11 and 3.12
 
 ## Quick Start
 

devo/__version__.py

@@ -1,6 +1,6 @@
 __description__ = "Devo Python Library."
 __url__ = "http://www.devo.com"
-__version__ = "5.4.1"
+__version__ = "6.0.0"
 __author__ = "Devo"
 __author_email__ = "[email protected]"
 __license__ = "MIT"

devo/api/client.py

@@ -55,6 +55,8 @@
     "connection_error": "Failed to establish a new connection",
     "other_errors": "Error while invoking query",
     "error_no_detail": "Error code %d while invoking query",
+    "no_keepalive_for_destination": "Queries with destination functionality only support No Keepalive mode. Forced to"
+    " NO_KEEP_ALIVE"
 }
 
 DEFAULT_KEEPALIVE_TOKEN = "\n"
@@ -173,6 +175,7 @@ def __init__(
         self.processor = None
         self.set_processor(processor)
         self.keepAliveToken = None
+
         self.set_keepalive_token(keepAliveToken)
 
         if pragmas:
@@ -235,7 +238,12 @@ def set_keepalive_token(self, keepAliveToken=DEFAULT_KEEPALIVE_TOKEN):
         # keepalive (cannot be modified), but implementation uses
         # NO_KEEP_ALIVE value as it does not change the query msgpack and
         # xls does not support keepalive
-        if self.response in [
+        # Queries with destination only supports NO_KEEP_ALIVE
+        if self.destination is not None:
+            self.keepAliveToken = NO_KEEPALIVE_TOKEN
+            if keepAliveToken not in [NO_KEEPALIVE_TOKEN, DEFAULT_KEEPALIVE_TOKEN]:
+                logging.warning(ERROR_MSGS["no_keepalive_for_destination"])
+        elif self.response in [
             "json",
             "json/compact",
             "json/simple",
@@ -254,7 +262,6 @@ def set_keepalive_token(self, keepAliveToken=DEFAULT_KEEPALIVE_TOKEN):
             self.keepAliveToken = NO_KEEPALIVE_TOKEN
         return True
 
-
 class Client:
     """
     The Devo search rest api main class
@@ -441,6 +448,7 @@ def query(
         :param limit: Max number of rows
         :param offset: start of needle for query
         :param comment: comment for query
+        :param ip_as_string: whether to recive IP types as strings
         :return: Result of the query (dict) or Iterator object
         """
         dates = self._generate_dates(dates)
@@ -755,18 +763,10 @@ def _generate_pragmas(self, comment=None):
 
         return str_pragmas
 
-    def get_jobs(self, job_type=None, name=None):
+    def get_jobs(self):
         """Get list of jobs by type and name, default All
-        :param job_type: category of jobs
-        :param name: name of jobs
         :return: json"""
-        plus = (
-            ""
-            if not job_type
-            else "/{}".format(job_type if not name else "{}/{}".format(job_type, name))
-        )
-
-        return self._call_jobs("{}{}{}".format(self.address[0], "/search/jobs", plus))
+        return self._call_jobs("{}{}".format(self.address[0], "/search/jobs"))
 
     def get_job(self, job_id):
         """Get all info of job

devo/api/scripts/client_cli.py

@@ -161,7 +161,7 @@ def configure(args):
     """
     Load CLI configuration
     :param args: args from files, launch vars, etc
-    :return: Clien  t API Object and Config values in array
+    :return: Client API Object and Config values in array
     """
     config = Configuration()
     try:

devo/common/dates/dateoperations.py

@@ -2,6 +2,9 @@
 """A collection of allowed operations on date parsing"""
 
 from datetime import datetime as dt
+import zoneinfo
+UTC = zoneinfo.ZoneInfo("UTC")
+
 from datetime import timedelta
 
 from .dateutils import to_millis, trunc_time, trunc_time_minute
@@ -60,31 +63,31 @@ def now():
     Return current millis in UTC
     :return: Millis
     """
-    return to_millis(dt.utcnow())
+    return to_millis(dt.now(UTC))
 
 
 def now_without_ms():
     """
     Return current millis in UTC
     :return: Millis
     """
-    return to_millis(trunc_time_minute(dt.utcnow()))
+    return to_millis(trunc_time_minute(dt.now(UTC)))
 
 
 def today():
     """
     Return current millis with the time truncated to 00:00:00
     :return: Millis
     """
-    return to_millis(trunc_time(dt.utcnow()))
+    return to_millis(trunc_time(dt.now(UTC)))
 
 
 def yesterday():
     """
     Return millis from yesterday with time truncated to 00:00:00
     :return: Millis
     """
-    return to_millis(trunc_time(dt.utcnow()) - timedelta(days=1))
+    return to_millis(trunc_time(dt.now(UTC)) - timedelta(days=1))
 
 
 def parse_functions():

devo/common/dates/dateutils.py

@@ -2,6 +2,8 @@
 """Utils for format and trunc dates."""
 
 from datetime import datetime as dt
+import zoneinfo
+UTC = zoneinfo.ZoneInfo("UTC")
 
 
 def to_millis(date):
@@ -10,7 +12,10 @@ def to_millis(date):
     :param date: Date for parse to millis
     :return: Millis from the date
     """
-    return int((date - dt.utcfromtimestamp(0)).total_seconds() * 1000)
+    # Verify whether param has timezone, if not set the default UTC one
+    if date.tzinfo is None:
+        date = date.replace(tzinfo=UTC)
+    return int((date - dt.fromtimestamp(0, UTC)).total_seconds() * 1000)
 
 
 def trunc_time(date):
@@ -50,4 +55,4 @@ def get_timestamp():
     Generate current timestamp
     :return:
     """
-    return to_millis(dt.utcnow())
+    return to_millis(dt.now(UTC))

devo/sender/data.py

@@ -15,6 +15,7 @@
 from ssl import SSLWantReadError, SSLWantWriteError
 from threading import Thread, Lock, Event
 from typing import Optional, Callable
+import warnings
 
 import pem
 from _socket import SHUT_WR
@@ -74,6 +75,7 @@ def __str__(self):
     RAW_SENDING_ERROR = ("Error sending raw event: %s",)
     CLOSING_ERROR = "Error closing connection"
     FLUSHING_BUFFER_ERROR = "Error flushing buffer"
+    ERROR_AFTER_TIMEOUT = "Timeout reached"
 
 
 class DevoSenderException(Exception):
@@ -593,8 +595,6 @@ def __connect_ssl(self):
                 context = ssl.create_default_context(cafile=self._sender_config.chain)
                 context.options |= ssl.OP_NO_SSLv2
                 context.options |= ssl.OP_NO_SSLv3
-                context.options |= ssl.OP_NO_TLSv1
-                context.options |= ssl.OP_NO_TLSv1_1
                 context.minimum_version = ssl.TLSVersion.TLSv1_2
                 context.maximum_version = ssl.TLSVersion.TLSv1_3
 
@@ -614,9 +614,16 @@ def __connect_ssl(self):
                     self.socket, server_hostname=self._sender_config.address[0]
                 )
             else:
-                self.socket = ssl.wrap_socket(
-                    self.socket, ssl_version=ssl.PROTOCOL_TLS, cert_reqs=ssl.CERT_NONE
-                )
+                context = ssl.create_default_context(ssl.Purpose.SERVER_AUTH)
+                context.check_hostname = False
+                context.verify_mode = ssl.CERT_NONE
+                context.options |= ssl.OP_NO_SSLv2
+                context.options |= ssl.OP_NO_SSLv3
+                context.minimum_version = ssl.TLSVersion.TLSv1_2
+                context.maximum_version = ssl.TLSVersion.TLSv1_3
+                self.logger.warning("One or more of CA certificate, private or public certificate is not provided"
+                                    " and TLS unsecure connection is established")
+                self.socket = context.wrap_socket(self.socket)
 
             self.socket.connect(self._sender_config.address)
             self.last_message = int(time.time())
@@ -643,17 +650,19 @@ def info(self, msg):
         """
         self.send(tag=self.logging.get("tag"), msg=msg)
 
-    # TODO: Deprecated
+    # TODO: Deprecated, to remove in next mayor version
     def set_sec_level(self, sec_level=None):
         """
         Set sec_level of SSL Context:
 
         :param sec_level: sec_level value
         :return
         """
+        warnings.warn("This function is deprecated and it will be removed in future versions",
+                      DeprecationWarning, stacklevel=2)
         self._sender_config.sec_level = sec_level
 
-    # TODO: Deprecated
+    # TODO: Deprecated, to remove in next mayor version
     def set_verify_mode(self, verify_mode=None):
         """
         Set verify_mode of SSL Context:
@@ -665,16 +674,20 @@ def set_verify_mode(self, verify_mode=None):
         :param verify_mode: verify mode value
         :return
         """
+        warnings.warn("This function is deprecated and it will be removed in future versions",
+                      DeprecationWarning, stacklevel=2)
         self._sender_config.verify_mode = verify_mode
 
-    # TODO: Deprecated
+    # TODO: Deprecated, to remove in next mayor version
     def set_check_hostname(self, check_hostname=True):
         """
         Set check_hostname of SSL Context:
 
         :param check_hostname: check_hostname value. Default True
         :return
         """
+        warnings.warn("This function is deprecated and it will be removed in future versions",
+                      DeprecationWarning, stacklevel=2)
         self._sender_config.check_hostname = check_hostname
 
     def buffer_size(self, size=19500):
@@ -1093,7 +1106,6 @@ def for_logging(config=None, con_type=None, tag=None, level=None):
         :param con_type: type of connection
         :param tag: tag for the table
         :param level: level of logger
-        :param formatter: log formatter
         :return: Sender object
         """
         con = Sender(config=config, con_type=con_type)

devo/sender/lookup.py

@@ -392,6 +392,7 @@ def field_to_str(field, escape_quotes=False):
         """
         Convert one value to STR, cleaning it
         :param field: field to clean
+        :param escape_quotes: whether to escape quotes in response
         :return:
         """
         return ",%s" % Lookup.clean_field(field, escape_quotes)
@@ -402,6 +403,7 @@ def process_fields(fields=None, key_index=None, escape_quotes=False):
         Method to convert list with one row/fields to STR to send
         :param fields: fields list
         :param key_index: index of key in fields
+        :param escape_quotes: whether to escape quotes in response
         :return:
         """
         # First the key
@@ -417,6 +419,7 @@ def clean_field(field=None, escape_quotes=False):
         """
         Strip and quotechar the fields
         :param str field: field for clean
+        :param escape_quotes: whether to escape quotes in response
         :return str: cleaned field
         """
         if not isinstance(field, (str, bytes)):

docs/api/api.md

@@ -496,6 +496,7 @@ Client support several modes for supporting this mechanism. The mode is set up i
   * `json`, `json/compact`, `json/simple` and `json/simple/compact` token is always `b'    '` (four utf-8 spaces chars)
   * For `csv` and `tsv` token is the custom `str` set as parameter
   * `msgpack` and `xls` do not support this mode
+* Queries using `destination` functionality does not support keep alive. NO_KEEP_ALIVE is forced
 
 | Response mode       | default mode              | `NO_KEEPALIVE_TOKEN` | `DEFAULT_KEEPALIVE_TOKEN` | `EMPTY_EVENT_KEEPALIVE_TOKEN` | Custom keep alive token      |
 |---------------------|---------------------------|----------------------|---------------------------|-------------------------------|------------------------------|

requirements-test.txt

@@ -1,7 +1,8 @@
-stopit==1.1.2
 msgpack~=1.0.8
 responses~=0.25.3
 pipdeptree~=2.23.0
 pytest~=8.2.2
 pytest-cov~=5.0.0
-mock==5.1.0
+mock==5.1.0
+pebble==5.0.7
+pytest-timeout~=2.3.1

requirements.txt

@@ -1,5 +1,5 @@
 click==8.1.7
-PyYAML==6.0.1
+PyYAML~=6.0.1
 requests~=2.32
 pem~=21.2.0
 pyopenssl~=24.2.1