Create new command for sftp upload

qualitytool/management/commands/save_daily_utilization_csv.py

@@ -0,0 +1,46 @@
+from django.core.management.base import BaseCommand
+from django.utils import timezone
+from django.conf import settings
+from datetime import datetime
+from qualitytool.models import ResourceQualityTool
+from qualitytool.manager import qt_manager
+import os.path as _
+import logging
+
+
+logger = logging.getLogger()
+
+
+
+class Command(BaseCommand):
+    help = "Saves daily utilization to a CSV file."
+
+    def add_arguments(self, parser):
+        parser.add_argument('--date', action='store')
+
+    def handle(self, *args, **options):
+        path = options.get('path')
+        date = options.get('date', None)
+
+
+        if date:
+            date = timezone.make_aware(datetime.strptime(date, '%Y-%m-%d'))
+        else:
+            date = timezone.now()
+
+
+        daily_utilizations = [
+            qt_manager.get_daily_utilization(qualitytool, date) 
+            for qualitytool in ResourceQualityTool.objects.all()
+        ]
+
+        if not daily_utilizations:
+            return
+
+
+        with open(path, 'w') as csv_file:
+            for daily_utilization in daily_utilizations:
+                csv_file.write('%(row)s\n' % ({
+                    'row': ','.join(str(val) for val in daily_utilization.values())
+                }))
+        logging.info(f'Generated new daily utilization csv file with {len(daily_utilizations)} entries.')

qualitytool/management/commands/sftp_daily_utilization.py

@@ -0,0 +1,60 @@
+from django.core.management.base import BaseCommand
+from django.core.exceptions import ImproperlyConfigured
+from django.utils import timezone
+from django.conf import settings
+from datetime import datetime
+from qualitytool.models import ResourceQualityTool
+from qualitytool.manager import qt_manager
+import os.path as _
+import logging
+import paramiko
+
+
+logger = logging.getLogger()
+
+class Command(BaseCommand):
+    help = "Uploads daily utilization to a sftp server."
+
+    def add_arguments(self, parser):
+        parser.add_argument('path')
+        parser.add_argument('--date', action='store')
+
+    def handle(self, *args, **options):
+        path = options.get('path')
+        date = options.get('date', None)
+
+        if not settings.QUALITYTOOL_SFTP_HOST:
+            raise ImproperlyConfigured('Missing env setting: QUALITYTOOL_SFTP_HOST')
+
+        if date:
+            date = timezone.make_aware(datetime.strptime(date, '%Y-%m-%d'))
+        else:
+            date = timezone.now()
+
+
+        daily_utilizations = [
+            qt_manager.get_daily_utilization(qualitytool, date) 
+            for qualitytool in ResourceQualityTool.objects.all()
+        ]
+
+        if not daily_utilizations:
+            return
+
+
+        transport = paramiko.Transport((settings.QUALITYTOOL_SFTP_HOST, settings.QUALITYTOOL_SFTP_PORT))
+
+        transport.connect(
+            username=settings.QUALITYTOOL_SFTP_USERNAME, 
+            password=settings.QUALITYTOOL_SFTP_PASSWORD)
+
+        sftp = paramiko.SFTPClient.from_transport(transport)
+
+
+
+
+        with sftp.open(path, 'w') as csv_file:
+            for daily_utilization in daily_utilizations:
+                csv_file.write('%(row)s\n' % ({
+                    'row': ','.join(str(val) for val in daily_utilization.values())
+                }))
+        logging.info(f'Generated new daily utilization csv file with {len(daily_utilizations)} entries.')

requirements.in

@@ -73,6 +73,7 @@ more-itertools
 ntlm-auth
 oauthlib
 packaging
+paramiko
 phonenumbers
 Pillow>7.0.0
 pluggy

requirements.txt

@@ -31,6 +31,8 @@ babel==2.11.0
     # via
     #   -r requirements.in
     #   delorean
+bcrypt==4.0.1
+    # via paramiko
 beautifulsoup4==4.11.2
     # via -r requirements.in
 cached-property==1.5.2
@@ -48,6 +50,7 @@ cffi==1.15.1
     # via
     #   -r requirements.in
     #   cryptography
+    #   pynacl
 chardet==5.1.0
     # via -r requirements.in
 charset-normalizer==3.0.1
@@ -71,6 +74,7 @@ cryptography==41.0.4
     # via
     #   -r requirements.in
     #   djangorestframework-simplejwt
+    #   paramiko
     #   pyjwt
     #   pyspnego
     #   requests-ntlm
@@ -204,6 +208,10 @@ ecdsa==0.18.0
     # via
     #   -r requirements.in
     #   python-jose
+exceptiongroup==1.1.3
+    # via
+    #   cattrs
+    #   pytest
 exchangelib==4.9.0
     # via -r requirements.in
 factory-boy==3.2.1
@@ -291,6 +299,8 @@ packaging==23.0
     #   deprecation
     #   drf-yasg
     #   pytest
+paramiko==3.3.1
+    # via -r requirements.in
 phonenumbers==8.13.6
     # via -r requirements.in
 pillow==10.0.1
@@ -353,6 +363,8 @@ pylint-plugin-utils==0.7
     # via
     #   -r requirements.in
     #   pylint-django
+pynacl==1.5.0
+    # via paramiko
 pyparsing==3.0.9
     # via -r requirements.in
 pyrsistent==0.19.3
@@ -444,7 +456,9 @@ ruamel-yaml==0.17.21
     #   -r requirements.in
     #   drf-yasg
 ruamel-yaml-clib==0.2.7
-    # via -r requirements.in
+    # via
+    #   -r requirements.in
+    #   ruamel-yaml
 sentry-sdk==1.15.0
     # via -r requirements.in
 six==1.16.0
@@ -477,13 +491,20 @@ tendo==0.3.0
     # via -r requirements.in
 text-unidecode==1.3
     # via -r requirements.in
+tomli==2.0.1
+    # via
+    #   coverage
+    #   pylint
+    #   pytest
 tomlkit==0.11.6
     # via pylint
 typed-ast==1.5.4
     # via -r requirements.in
 typing-extensions==4.5.0
     # via
+    #   astroid
     #   django-modeltranslation
+    #   pylint
     #   swagger-spec-validator
 tzdata==2022.7
     # via

respa/settings.py

@@ -106,6 +106,10 @@
     QUALITYTOOL_PASSWORD=(str, ''),
     QUALITYTOOL_API_BASE=(str, ''),
     QUALITYTOOL_ENABLED=(bool, False),
+    QUALITYTOOL_SFTP_HOST=(str, ''),
+    QUALITYTOOL_SFTP_PORT=(int, 22),
+    QUALITYTOOL_SFTP_USERNAME=(str, ''),
+    QUALITYTOOL_SFTP_PASSWORD=(str, '')
 )
 environ.Env.read_env()
 # used for generating links to images, when no request context is available
@@ -171,6 +175,11 @@
 QUALITYTOOL_API_BASE = env('QUALITYTOOL_API_BASE')
 QUALITYTOOL_ENABLED = env('QUALITYTOOL_ENABLED')
 
+QUALITYTOOL_SFTP_HOST = env('QUALITYTOOL_SFTP_HOST')
+QUALITYTOOL_SFTP_PORT = env('QUALITYTOOL_SFTP_PORT')
+QUALITYTOOL_SFTP_USERNAME = env('QUALITYTOOL_SFTP_USERNAME')
+QUALITYTOOL_SFTP_PASSWORD = env('QUALITYTOOL_SFTP_PASSWORD')
+
 # Application definition
 INSTALLED_APPS = [
     'helusers',