Merge pull request #201 from City-of-Helsinki/develop

Release 20210118

leasing/report/lease/lease_statistic_report.py

@@ -502,6 +502,17 @@ def get_data(self, input_data):
                 Q(end_date__isnull=True) | Q(end_date__gte=datetime.date.today())
             )
 
+        # Skip the leases where have not set the period type of base amount in contact rents
+        # The report fails because contract rents contain items where isn't defined the period type for the base amount
+        # TODO: Review this with the specialist
+        no_period = []
+        for lease in qs:
+            for rent in lease.rents.all():
+                for cr in rent.contract_rents.all():
+                    if not cr.base_amount_period:
+                        no_period.append(lease.id)
+        qs = qs.exclude(id__in=no_period)
+
         return qs
 
     def generate_report(self, user, input_data):

leasing/report/report_base.py

@@ -68,8 +68,8 @@ class ReportBase:
     # If the column labels should be automatically added as the first row
     automatic_excel_column_labels = True
 
-    def __init__(self):
-        self.form = None
+    # The query form model of report
+    form = None
 
     @classmethod
     def get_output_fields_metadata(cls):
@@ -88,6 +88,10 @@ def get_form(self, data=None):
         self.form instance attribute and returns it."""
         self.form = ReportFormBase(data, input_fields=self.input_fields)
 
+        # This has been set to None as the report doesn't require any form rendering
+        # and it causes pickle error in Django Q async tasks.
+        self.form.renderer = None
+
         return self.form
 
     def get_input_data(self, request):

leasing/tests/report/test_reports.py

@@ -0,0 +1,57 @@
+from multiprocessing import Event, Value
+
+import pytest
+from django.core import mail
+from django_q.brokers import get_broker
+from django_q.cluster import monitor, pusher, worker
+from django_q.queues import Queue
+from django_q.tasks import queue_size
+
+from leasing.report.lease.lease_statistic_report import LeaseStatisticReport
+
+
+@pytest.fixture(autouse=True)
+def use_q_cluster_testing(settings):
+    settings.Q_CLUSTER = {
+        "name": "DjangORM",
+        "cpu_affinity": 1,
+        "testing": True,
+        "log_level": "DEBUG",
+        "orm": "default",
+    }
+
+
+@pytest.mark.django_db(transaction=True)
+def test_simple_async_report_send(rf, admin_user):
+    broker = get_broker()
+    assert broker.queue_size() == 0
+
+    request = rf.get("/")
+    request.query_params = {}
+    request.user = admin_user
+
+    report = LeaseStatisticReport()
+    response = report.get_response(request)
+    assert response.data
+    assert broker.queue_size() == 1
+
+    # Run async task
+    task_queue = Queue()
+    result_queue = Queue()
+    event = Event()
+    event.set()
+    pusher(task_queue, event, broker=broker)
+    assert task_queue.qsize() == 1
+    assert queue_size(broker=broker) == 0
+    task_queue.put("STOP")
+    worker(task_queue, result_queue, Value("f", -1))
+    assert task_queue.qsize() == 0
+    assert result_queue.qsize() == 1
+    result_queue.put("STOP")
+    monitor(result_queue)
+    assert result_queue.qsize() == 0
+    broker.delete_queue()
+
+    # Test report file have been sent via email
+    assert len(mail.outbox) == 1
+    assert len(mail.outbox[0].attachments) == 1

local_settings.py.template

@@ -0,0 +1,11 @@
+Q_CLUSTER = {
+    "name": "DjangORM",
+    "timeout": 90,
+    "retry": 60 * 60,  # 1 hour
+    "orm": "default",
+    "error_reporter": {
+        "sentry": {
+            "dsn": "https://******@sentry.io/<project>"
+        }
+    }
+}

mvj/settings.py

@@ -149,11 +149,11 @@ def get_git_revision_hash():
     "laske_export",
     "field_permissions",
     "batchrun",
-    "django_q",
     "constance",
     "constance.backends.database",
     "sanitized_dump",
     "utils",
+    "django_q",
 ]
 
 if DEBUG:

requirements-dev.txt

@@ -4,70 +4,179 @@
 #
 #    pip-compile requirements-dev.in
 #
-appdirs==1.4.3            # via -c requirements.txt, black
-attrs==19.3.0             # via -c requirements.txt, black, pytest
-backcall==0.1.0           # via ipython
-black==19.10b0            # via -r requirements-dev.in
-click==7.1.2              # via black
-coverage==5.0.4           # via pytest-cov
-debugpy==1.1.0            # via -r requirements-dev.in
-decorator==4.4.2          # via ipython, traitlets
-django-debug-toolbar==2.2  # via -r requirements-dev.in
-django-extensions==2.2.9  # via -r requirements-dev.in
-django-stubs==1.5.0       # via -r requirements-dev.in
-django==2.2.13            # via -c requirements.txt, django-debug-toolbar, django-stubs
-entrypoints==0.3          # via flake8
-et-xmlfile==1.0.1         # via openpyxl
-factory-boy==2.12.0       # via -r requirements-dev.in, pytest-factoryboy
-faker==4.0.2              # via factory-boy
-flake8-polyfill==1.0.2    # via pep8-naming
-flake8-print==3.1.4       # via -r requirements-dev.in
-flake8==3.7.9             # via -r requirements-dev.in, flake8-polyfill, flake8-print
-importlib-metadata==2.0.0  # via pluggy, pytest
-inflection==0.3.1         # via pytest-factoryboy
-ipython-genutils==0.2.0   # via traitlets
-ipython==7.13.0           # via -r requirements-dev.in
-isort==5.6.4              # via -r requirements-dev.in
-jdcal==1.4.1              # via openpyxl
-jedi==0.16.0              # via ipython
-mccabe==0.6.1             # via flake8
-more-itertools==8.2.0     # via pytest
-mypy-extensions==0.4.3    # via mypy
-mypy==0.770               # via -r requirements-dev.in, django-stubs
-openpyxl==3.0.3           # via -r requirements-dev.in
-packaging==20.3           # via pytest
-parso==0.6.2              # via jedi
-pathspec==0.8.0           # via black
-pep8-naming==0.10.0       # via -r requirements-dev.in
-pexpect==4.8.0            # via ipython
-pickleshare==0.7.5        # via ipython
-pluggy==0.13.1            # via pytest
-prompt-toolkit==3.0.5     # via ipython
-ptyprocess==0.6.0         # via pexpect
-py==1.8.1                 # via pytest
-pycodestyle==2.5.0        # via flake8, flake8-print
-pydocstyle==5.0.2         # via -r requirements-dev.in
-pyflakes==2.1.1           # via flake8
-pygments==2.6.1           # via ipython
-pyparsing==2.4.6          # via packaging
-pytest-cov==2.8.1         # via -r requirements-dev.in
-pytest-django==3.9.0      # via -r requirements-dev.in
-pytest-factoryboy==2.0.3  # via -r requirements-dev.in
-pytest==5.4.1             # via -r requirements-dev.in, pytest-cov, pytest-django, pytest-factoryboy
-python-dateutil==2.6.0    # via -c requirements.txt, faker
-pytz==2019.3              # via -c requirements.txt, django
-regex==2020.5.14          # via black
-six==1.14.0               # via -c requirements.txt, django-extensions, flake8-print, packaging, python-dateutil, traitlets
-snowballstemmer==2.0.0    # via pydocstyle
-sqlparse==0.3.1           # via -c requirements.txt, django, django-debug-toolbar
-text-unidecode==1.3       # via faker
-toml==0.10.1              # via black
-traitlets==4.3.3          # via ipython
-typed-ast==1.4.1          # via black, mypy
-typing-extensions==3.7.4.2  # via django-stubs, mypy
-wcwidth==0.1.9            # via -c requirements.txt, prompt-toolkit, pytest
-werkzeug==1.0.1           # via -r requirements-dev.in
-zipp==3.3.0               # via importlib-metadata
+appdirs==1.4.3
+    # via
+    #   -c requirements.txt
+    #   black
+attrs==19.3.0
+    # via
+    #   -c requirements.txt
+    #   black
+    #   pytest
+backcall==0.1.0
+    # via ipython
+black==19.10b0
+    # via -r requirements-dev.in
+click==7.1.2
+    # via black
+coverage==5.0.4
+    # via pytest-cov
+debugpy==1.1.0
+    # via -r requirements-dev.in
+decorator==4.4.2
+    # via
+    #   ipython
+    #   traitlets
+django-debug-toolbar==2.2
+    # via -r requirements-dev.in
+django-extensions==2.2.9
+    # via -r requirements-dev.in
+django-stubs==1.5.0
+    # via -r requirements-dev.in
+django==2.2.13
+    # via
+    #   -c requirements.txt
+    #   django-debug-toolbar
+    #   django-stubs
+entrypoints==0.3
+    # via flake8
+et-xmlfile==1.0.1
+    # via openpyxl
+factory-boy==2.12.0
+    # via
+    #   -r requirements-dev.in
+    #   pytest-factoryboy
+faker==4.0.2
+    # via factory-boy
+flake8-polyfill==1.0.2
+    # via pep8-naming
+flake8-print==3.1.4
+    # via -r requirements-dev.in
+flake8==3.7.9
+    # via
+    #   -r requirements-dev.in
+    #   flake8-polyfill
+    #   flake8-print
+importlib-metadata==2.0.0
+    # via
+    #   pluggy
+    #   pytest
+inflection==0.3.1
+    # via pytest-factoryboy
+ipython-genutils==0.2.0
+    # via traitlets
+ipython==7.13.0
+    # via -r requirements-dev.in
+isort==5.6.4
+    # via -r requirements-dev.in
+jdcal==1.4.1
+    # via openpyxl
+jedi==0.16.0
+    # via ipython
+mccabe==0.6.1
+    # via flake8
+more-itertools==8.2.0
+    # via pytest
+mypy-extensions==0.4.3
+    # via mypy
+mypy==0.770
+    # via
+    #   -r requirements-dev.in
+    #   django-stubs
+openpyxl==3.0.3
+    # via -r requirements-dev.in
+packaging==20.3
+    # via pytest
+parso==0.6.2
+    # via jedi
+pathspec==0.8.0
+    # via black
+pep8-naming==0.10.0
+    # via -r requirements-dev.in
+pexpect==4.8.0
+    # via ipython
+pickleshare==0.7.5
+    # via ipython
+pluggy==0.13.1
+    # via pytest
+prompt-toolkit==3.0.5
+    # via ipython
+ptyprocess==0.6.0
+    # via pexpect
+py==1.8.1
+    # via pytest
+pycodestyle==2.5.0
+    # via
+    #   flake8
+    #   flake8-print
+pydocstyle==5.0.2
+    # via -r requirements-dev.in
+pyflakes==2.1.1
+    # via flake8
+pygments==2.6.1
+    # via ipython
+pyparsing==2.4.6
+    # via packaging
+pytest-cov==2.8.1
+    # via -r requirements-dev.in
+pytest-django==3.9.0
+    # via -r requirements-dev.in
+pytest-factoryboy==2.0.3
+    # via -r requirements-dev.in
+pytest==5.4.1
+    # via
+    #   -r requirements-dev.in
+    #   pytest-cov
+    #   pytest-django
+    #   pytest-factoryboy
+python-dateutil==2.6.0
+    # via
+    #   -c requirements.txt
+    #   faker
+pytz==2019.3
+    # via
+    #   -c requirements.txt
+    #   django
+regex==2020.5.14
+    # via black
+six==1.14.0
+    # via
+    #   -c requirements.txt
+    #   django-extensions
+    #   flake8-print
+    #   packaging
+    #   python-dateutil
+    #   traitlets
+snowballstemmer==2.0.0
+    # via pydocstyle
+sqlparse==0.3.1
+    # via
+    #   -c requirements.txt
+    #   django
+    #   django-debug-toolbar
+text-unidecode==1.3
+    # via faker
+toml==0.10.1
+    # via black
+traitlets==4.3.3
+    # via ipython
+typed-ast==1.4.1
+    # via
+    #   black
+    #   mypy
+typing-extensions==3.7.4.2
+    # via
+    #   django-stubs
+    #   mypy
+wcwidth==0.1.9
+    # via
+    #   -c requirements.txt
+    #   prompt-toolkit
+    #   pytest
+werkzeug==1.0.1
+    # via -r requirements-dev.in
+zipp==3.3.0
+    # via importlib-metadata
 
 # The following packages are considered to be unsafe in a requirements file:
 # setuptools

requirements.in

@@ -15,7 +15,7 @@ django-filter
 django-helusers
 django-model-utils
 django-modeltranslation
-django-q
+django-q[sentry]
 django-safedelete
 django-sanitized-dump
 django-sequences