[monitoring] Adding influxDB 2.x version support #274

.github/workflows/ci.yml

@@ -62,7 +62,7 @@ jobs:
           pip install -U pip wheel setuptools
 
       - name: Install npm dependencies
-        run: sudo npm install -g install jshint stylelint
+        run: sudo npm install -g jshint stylelint
 
       - name: Start InfluxDB container
         run: docker-compose up -d influxdb

Dockerfile

@@ -1,13 +1,16 @@
 FROM python:3.9.19-slim-bullseye
 
+# Install system dependencies
 RUN apt update && \
     apt install --yes zlib1g-dev libjpeg-dev gdal-bin libproj-dev \
     libgeos-dev libspatialite-dev libsqlite3-mod-spatialite \
     sqlite3 libsqlite3-dev openssl libssl-dev fping && \
     rm -rf /var/lib/apt/lists/* /root/.cache/pip/* /tmp/*
 
+# Upgrade pip and install Python dependencies
 RUN pip install -U pip setuptools wheel
 
+# Copy and install project dependencies
 COPY requirements-test.txt requirements.txt /opt/openwisp/
 RUN pip install -r /opt/openwisp/requirements.txt && \
     pip install -r /opt/openwisp/requirements-test.txt && \
@@ -17,9 +20,11 @@ ADD . /opt/openwisp
 RUN pip install -U /opt/openwisp && \
     rm -rf /var/lib/apt/lists/* /root/.cache/pip/* /tmp/*
 WORKDIR /opt/openwisp/tests/
+# Set environment variables
 ENV NAME=openwisp-monitoring \
     PYTHONBUFFERED=1 \
     INFLUXDB_HOST=influxdb \
     REDIS_HOST=redis
 CMD ["sh", "docker-entrypoint.sh"]
 EXPOSE 8000
+# Command to run the application

README.rst

@@ -326,20 +326,23 @@ Follow the setup instructions of `openwisp-controller
 
     # Make sure you change them in production
     # You can select one of the backends located in openwisp_monitoring.db.backends
-    TIMESERIES_DATABASE = {
+    INFLUXDB_1x_DATABASE = {
         'BACKEND': 'openwisp_monitoring.db.backends.influxdb',
         'USER': 'openwisp',
         'PASSWORD': 'openwisp',
         'NAME': 'openwisp2',
-        'HOST': 'localhost',
+        'HOST': 'influxdb',
         'PORT': '8086',
-        'OPTIONS': {
-            # Specify additional options to be used while initializing
-            # database connection.
-            # Note: These options may differ based on the backend used.
-            'udp_writes': True,
-            'udp_port': 8089,
-        }
+        'OPTIONS': {'udp_writes': False, 'udp_port': 8089},
+    }   
+
+    INFLUXDB_2x_DATABASE = {
+        'BACKEND': 'openwisp_monitoring.db.backends.influxdb2',
+        'TOKEN': 'my-super-secret-auth-token',
+        'ORG': 'openwisp', 
+        'BUCKET': 'openwisp2',
+        'HOST': 'influxdb2',
+        'PORT': '9999',
     }
 
 ``urls.py``:
@@ -1413,56 +1416,109 @@ Settings
 | **default**: | see below |
 +--------------+-----------+
 
+Timeseries Database Configuration
+---------------------------------
+
+The ``TIMESERIES_DATABASE`` setting allows configuring the timeseries 
+database backend used by OpenWISP Monitoring. The configuration supports 
+both InfluxDB 1.x and 2.x versions.
+
+Configuration for InfluxDB 1.x
+------------------------------
+
 .. code-block:: python
 
-    TIMESERIES_DATABASE = {
+    INFLUXDB_1x_DATABASE = {
         'BACKEND': 'openwisp_monitoring.db.backends.influxdb',
         'USER': 'openwisp',
         'PASSWORD': 'openwisp',
         'NAME': 'openwisp2',
-        'HOST': 'localhost',
+        'HOST': 'influxdb',
         'PORT': '8086',
-        'OPTIONS': {
-            'udp_writes': False,
-            'udp_port': 8089,
-        }
+        'OPTIONS': {'udp_writes': False, 'udp_port': 8089},
     }
 
-The following table describes all keys available in ``TIMESERIES_DATABASE``
-setting:
-
-+---------------+--------------------------------------------------------------------------------------+
-| **Key**       | ``Description``                                                                      |
-+---------------+--------------------------------------------------------------------------------------+
-| ``BACKEND``   | The timeseries database backend to use. You can select one of the backends           |
-|               | located in ``openwisp_monitoring.db.backends``                                       |
-+---------------+--------------------------------------------------------------------------------------+
-| ``USER``      | User for logging into the timeseries database                                        |
-+---------------+--------------------------------------------------------------------------------------+
-| ``PASSWORD``  | Password of the timeseries database user                                             |
-+---------------+--------------------------------------------------------------------------------------+
-| ``NAME``      | Name of the timeseries database                                                      |
-+---------------+--------------------------------------------------------------------------------------+
-| ``HOST``      | IP address/hostname of machine where the timeseries database is running              |
-+---------------+--------------------------------------------------------------------------------------+
-| ``PORT``      | Port for connecting to the timeseries database                                       |
-+---------------+--------------------------------------------------------------------------------------+
-| ``OPTIONS``   | These settings depends on the timeseries backend:                                    |
-|               |                                                                                      |
-|               | +-----------------+----------------------------------------------------------------+ |
-|               | | ``udp_writes``  | Whether to use UDP for writing data to the timeseries database | |
-|               | +-----------------+----------------------------------------------------------------+ |
-|               | | ``udp_port``    | Timeseries database port for writing data using UDP            | |
-|               | +-----------------+----------------------------------------------------------------+ |
-+---------------+--------------------------------------------------------------------------------------+
-
-**Note:** UDP packets can have a maximum size of 64KB. When using UDP for writing timeseries
-data, if the size of the data exceeds 64KB, TCP mode will be used instead.
-
-**Note:** If you want to use the ``openwisp_monitoring.db.backends.influxdb`` backend
-with UDP writes enabled, then you need to enable two different ports for UDP
-(each for different retention policy) in your InfluxDB configuration. The UDP configuration
-section of your InfluxDB should look similar to the following:
+Configuration for InfluxDB 2.x
+------------------------------
+
+.. code-block:: python
+
+    INFLUXDB_2x_DATABASE = {
+        'BACKEND': 'openwisp_monitoring.db.backends.influxdb2',
+        'TOKEN': 'my-super-secret-auth-token',
+        'ORG': 'openwisp',
+        'BUCKET': 'openwisp2',
+        'HOST': 'influxdb2',
+        'PORT': '9999',
+    }
+
+Dynamic Configuration Based on Environment
+------------------------------------------
+
+You can dynamically switch between InfluxDB 1.x and 2.x configurations 
+using environment variables:
+
+.. code-block:: python
+
+    import os
+
+    if os.environ.get('USE_INFLUXDB2', 'False') == 'True':
+        TIMESERIES_DATABASE = INFLUXDB_2x_DATABASE
+    else:
+        TIMESERIES_DATABASE = INFLUXDB_1x_DATABASE
+
+    if TESTING:
+        if os.environ.get('TIMESERIES_UDP', False):
+            TIMESERIES_DATABASE['OPTIONS'] = {'udp_writes': True, 'udp_port': 8091}
+
+Explanation of Settings
+-----------------------
+
++---------------+---------------------------------------------------------------+
+| **Key**       | **Description**                                               |
++-------------------------------------------------------------------------------+
+| ``BACKEND``   | The timeseries database backend to use. You can select one    |
+|               | of the backends located in ``openwisp_monitoring.db.backends``|
++---------------+---------------------------------------------------------------+
+| ``USER``      | User for logging into the timeseries database (only for       |
+|               | InfluxDB 1.x)                                                 |
++---------------+---------------------------------------------------------------+
+| ``PASSWORD``  | Password of the timeseries database user (only for InfluxDB   |
+|               | 1.x)                                                          |
++---------------+---------------------------------------------------------------+
+| ``NAME``      | Name of the timeseries database (only for InfluxDB 1.x)       |
++---------------+---------------------------------------------------------------+
+| ``TOKEN``     | Authentication token for InfluxDB 2.x                         |
++---------------+---------------------------------------------------------------+
+| ``ORG``       | Organization name for InfluxDB 2.x                            |
++---------------+---------------------------------------------------------------+
+| ``BUCKET``    | Bucket name for InfluxDB 2.x                                  |
++---------------+---------------------------------------------------------------+
+| ``HOST``      | IP address/hostname of machine where the timeseries           |
+|               | database is running                                           |
++---------------+---------------------------------------------------------------+
+| ``PORT``      | Port for connecting to the timeseries database                |
++---------------+---------------------------------------------------------------+
+| ``OPTIONS``   | Additional options for the timeseries backend                 |
+|               |                                                               |
+|               | +-----------------+-----------------------------------------+ |
+|               | | ``udp_writes``  | Whether to use UDP for writing data     | |
+|               | |                 | to the timeseries database              | |
+|               | +-----------------+-----------------------------------------+ |
+|               | | ``udp_port``    | Timeseries database port for writing    | |
+|               | |                 | data using UDP                          | |
+|               | +-----------------+-----------------------------------------+ |
++---------------+---------------------------------------------------------------+
+
+UDP Configuration for InfluxDB 1.x
+----------------------------------
+
+If you want to use the ``openwisp_monitoring.db.backends.influxdb`` backend 
+with UDP writes enabled, you need to enable two different ports for UDP 
+(each for a different retention policy) in your InfluxDB configuration.
+
+Here is an example of the UDP configuration section in your InfluxDB 
+configuration file:
 
 .. code-block:: text
 
@@ -1479,6 +1535,13 @@ section of your InfluxDB should look similar to the following:
     database = "openwisp2"
     retention-policy = 'short'
 
+**Note:** UDP packets can have a maximum size of 64KB. When using UDP for 
+writing timeseries data, if the size of the data exceeds 64KB, TCP mode 
+will be used instead.
+
+Deploying with Ansible
+----------------------
+
 If you are using `ansible-openwisp2 <https://github.com/openwisp/ansible-openwisp2>`_
 for deploying OpenWISP, you can set the ``influxdb_udp_mode`` ansible variable to ``true``
 in your playbook, this will make the ansible role automatically configure the InfluxDB UDP listeners.

docker-compose.yml

@@ -28,6 +28,22 @@ services:
       INFLUXDB_USER: openwisp
       INFLUXDB_USER_PASSWORD: openwisp
 
+  influxdb2:
+    image: influxdb:2.0
+    container_name: influxdb2
+    ports:
+      # Map the 9086 port on host machine to 8086 in container
+      - "9086:8086"
+    environment:
+      DOCKER_INFLUXDB_INIT_MODE: setup
+      DOCKER_INFLUXDB_INIT_USERNAME: myuser
+      DOCKER_INFLUXDB_INIT_PASSWORD: mypassword
+      DOCKER_INFLUXDB_INIT_ORG: myorg
+      DOCKER_INFLUXDB_INIT_BUCKET: mybucket
+      DOCKER_INFLUXDB_INIT_RETENTION: 1w
+    volumes:
+      - influxdb-storage:/var/lib/influxdb2
+
   redis:
     image: redis:5.0-alpine
     ports:
@@ -36,3 +52,4 @@ services:
 
 volumes:
   influxdb-data: {}
+  influxdb-storage:

openwisp_monitoring/db/backends/__init__.py

@@ -9,19 +9,36 @@
 
 TIMESERIES_DB = getattr(settings, 'TIMESERIES_DATABASE', None)
 if not TIMESERIES_DB:
-    TIMESERIES_DB = {
-        'BACKEND': 'openwisp_monitoring.db.backends.influxdb',
-        'USER': getattr(settings, 'INFLUXDB_USER', 'openwisp'),
-        'PASSWORD': getattr(settings, 'INFLUXDB_PASSWORD', 'openwisp'),
-        'NAME': getattr(settings, 'INFLUXDB_DATABASE', 'openwisp2'),
-        'HOST': getattr(settings, 'INFLUXDB_HOST', 'localhost'),
-        'PORT': getattr(settings, 'INFLUXDB_PORT', '8086'),
-    }
-    logger.warning(
-        'The previous method to define Timeseries Database has been deprecated. Please refer to the docs:\n'
-        'https://github.com/openwisp/openwisp-monitoring#setup-integrate-in-an-existing-django-project'
-    )
+    INFLUXDB_BACKEND = getattr(settings, 'INFLUXDB_BACKEND', 'openwisp_monitoring.db.backends.influxdb')
+
+    if INFLUXDB_BACKEND == 'openwisp_monitoring.db.backends.influxdb':
+        # InfluxDB 1.x configuration
+        TIMESERIES_DB = {
+            'BACKEND': INFLUXDB_BACKEND,
+            'USER': getattr(settings, 'INFLUXDB_USER', 'openwisp'),
+            'PASSWORD': getattr(settings, 'INFLUXDB_PASSWORD', 'openwisp'),
+            'NAME': getattr(settings, 'INFLUXDB_DATABASE', 'openwisp2'),
+            'HOST': getattr(settings, 'INFLUXDB_HOST', 'localhost'),
+            'PORT': getattr(settings, 'INFLUXDB_PORT', '8086'),
+        }
+    elif INFLUXDB_BACKEND == 'openwisp_monitoring.db.backends.influxdb2':
+        # InfluxDB 2.x configuration
+        TIMESERIES_DB = {
+            'BACKEND': INFLUXDB_BACKEND,
+            'TOKEN': getattr(settings, 'INFLUXDB_TOKEN', 'dltiEmsmMKU__9SoBE0ingFdMTS3UksrESwIQDNtW_3WOgn8bQGdyYzPcx_aDtvZkqvR8RbMkwVVlzUJxpm62w=='),
+            'ORG': getattr(settings, 'INFLUXDB_ORG', 'myorg'),
+            'BUCKET': getattr(settings, 'INFLUXDB_BUCKET', 'mybucket'),
+            'HOST': getattr(settings, 'INFLUXDB_HOST', 'localhost'),
+            'PORT': getattr(settings, 'INFLUXDB_PORT', '8086'),
+        }
+    else:
+        logger.warning('Invalid INFLUXDB_BACKEND setting. Please check the documentation.')
 
+    if INFLUXDB_BACKEND == 'openwisp_monitoring.db.backends.influxdb':
+        logger.warning(
+            'The previous method to define Timeseries Database has been deprecated. Please refer to the docs:\n'
+            'https://github.com/openwisp/openwisp-monitoring#setup-integrate-in-an-existing-django-project'
+        )
 
 def load_backend_module(backend_name=TIMESERIES_DB['BACKEND'], module=None):
     """
@@ -30,9 +47,17 @@ def load_backend_module(backend_name=TIMESERIES_DB['BACKEND'], module=None):
     """
     try:
         assert 'BACKEND' in TIMESERIES_DB, 'BACKEND'
-        assert 'USER' in TIMESERIES_DB, 'USER'
-        assert 'PASSWORD' in TIMESERIES_DB, 'PASSWORD'
-        assert 'NAME' in TIMESERIES_DB, 'NAME'
+        is_influxdb2 = '2' in TIMESERIES_DB['BACKEND']
+        if is_influxdb2:
+            # InfluxDB 2.x specific checks
+            assert 'TOKEN' in TIMESERIES_DB, 'TOKEN'
+            assert 'ORG' in TIMESERIES_DB, 'ORG'
+            assert 'BUCKET' in TIMESERIES_DB, 'BUCKET'
+        else:
+            # InfluxDB 1.x specific checks
+            assert 'USER' in TIMESERIES_DB, 'USER'
+            assert 'PASSWORD' in TIMESERIES_DB, 'PASSWORD'
+            assert 'NAME' in TIMESERIES_DB, 'NAME'
         assert 'HOST' in TIMESERIES_DB, 'HOST'
         assert 'PORT' in TIMESERIES_DB, 'PORT'
         if module:
@@ -48,7 +73,7 @@ def load_backend_module(backend_name=TIMESERIES_DB['BACKEND'], module=None):
     except ImportError as e:
         # The database backend wasn't found. Display a helpful error message
         # listing all built-in database backends.
-        builtin_backends = ['influxdb']
+        builtin_backends = ['influxdb', 'influxdb2']
         if backend_name not in [
             f'openwisp_monitoring.db.backends.{b}' for b in builtin_backends
         ]:
@@ -57,7 +82,18 @@ def load_backend_module(backend_name=TIMESERIES_DB['BACKEND'], module=None):
                 "Try using 'openwisp_monitoring.db.backends.XXX', where XXX is one of:\n"
                 f"{builtin_backends}"
             ) from e
+        else:
+            raise e
 
 
-timeseries_db = load_backend_module(module='client').DatabaseClient()
-timeseries_db.queries = load_backend_module(module='queries')
+if '2' in TIMESERIES_DB['BACKEND']:
+    timeseries_db = load_backend_module(module='client').DatabaseClient(
+        bucket=TIMESERIES_DB['BUCKET'],
+        org=TIMESERIES_DB['ORG'],
+        token=TIMESERIES_DB['TOKEN'],
+        url=f"http://{TIMESERIES_DB['HOST']}:{TIMESERIES_DB['PORT']}",
+    )
+    timeseries_db.queries = load_backend_module(TIMESERIES_DB['BACKEND'], module='queries')
+else:
+    timeseries_db = load_backend_module(module='client').DatabaseClient()
+    timeseries_db.queries = load_backend_module(module='queries')