From fd7c5ff4849444da9b3f80f6c5b433bfcd249cb5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 4 Oct 2023 01:33:02 +0000 Subject: [PATCH 1/7] Bump pillow from 9.5.0 to 10.0.1 Bumps [pillow](https://github.com/python-pillow/Pillow) from 9.5.0 to 10.0.1. - [Release notes](https://github.com/python-pillow/Pillow/releases) - [Changelog](https://github.com/python-pillow/Pillow/blob/main/CHANGES.rst) - [Commits](https://github.com/python-pillow/Pillow/compare/9.5.0...10.0.1) --- updated-dependencies: - dependency-name: pillow dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements.in | 2 +- requirements.txt | 8 +------- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/requirements.in b/requirements.in index 6b9e4ec0e2dc..5efec804add5 100644 --- a/requirements.in +++ b/requirements.in @@ -37,7 +37,7 @@ drf-spectacular # DRF API documentation feedparser # RSS newsfeed parser gunicorn # Gunicorn web server pdf2image # PDF to image conversion -pillow==9.5.0 # Image manipulation # FIXED 2023-07-04 as we require PIL.Image.ANTIALIAS +pillow==10.0.1 # Image manipulation # FIXED 2023-07-04 as we require PIL.Image.ANTIALIAS pint==0.21 # Unit conversion # FIXED 2023-05-30 breaks tests https://github.com/matmair/InvenTree/actions/runs/5095665936/jobs/9160852560 python-barcode[images] # Barcode generator python-dotenv # Environment variable management diff --git a/requirements.txt b/requirements.txt index 0ef5bd82ac17..055b542f7136 100644 --- a/requirements.txt +++ b/requirements.txt @@ -171,10 +171,6 @@ icalendar==5.0.7 # via django-ical idna==3.4 # via requests -importlib-metadata==6.8.0 - # via - # django-q2 - # markdown inflection==0.5.1 # via drf-spectacular itypes==1.2.0 @@ -201,7 +197,7 @@ packaging==23.1 # via gunicorn pdf2image==1.16.3 # via -r requirements.in -pillow==9.5.0 +pillow==10.0.1 # via # -r requirements.in # django-stdimage @@ -326,8 +322,6 @@ xlrd==2.0.1 # via tablib xlwt==1.3.0 # via tablib -zipp==3.16.0 - # via importlib-metadata zopfli==0.2.2 # via fonttools From da9969f49fa4cf99d1a5f7b79b6340a774299a07 Mon Sep 17 00:00:00 2001 From: Oliver Date: Wed, 4 Oct 2023 14:09:21 +1100 Subject: [PATCH 2/7] Patch requirements files --- requirements.in | 2 +- requirements.txt | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/requirements.in b/requirements.in index 5efec804add5..f90853ffa691 100644 --- a/requirements.in +++ b/requirements.in @@ -37,7 +37,7 @@ drf-spectacular # DRF API documentation feedparser # RSS newsfeed parser gunicorn # Gunicorn web server pdf2image # PDF to image conversion -pillow==10.0.1 # Image manipulation # FIXED 2023-07-04 as we require PIL.Image.ANTIALIAS +pillow>=10.0.1 # Image manipulation pint==0.21 # Unit conversion # FIXED 2023-05-30 breaks tests https://github.com/matmair/InvenTree/actions/runs/5095665936/jobs/9160852560 python-barcode[images] # Barcode generator python-dotenv # Environment variable management diff --git a/requirements.txt b/requirements.txt index 055b542f7136..a5176bfd0b52 100644 --- a/requirements.txt +++ b/requirements.txt @@ -171,6 +171,10 @@ icalendar==5.0.7 # via django-ical idna==3.4 # via requests +importlib-metadata==6.8.0 + # via + # django-q2 + # markdown inflection==0.5.1 # via drf-spectacular itypes==1.2.0 @@ -322,6 +326,8 @@ xlrd==2.0.1 # via tablib xlwt==1.3.0 # via tablib +zipp==3.16.0 + # via importlib-metadata zopfli==0.2.2 # via fonttools From 583c4ce4b359e0a57550cbc12f6a5de564f93345 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Thu, 19 Oct 2023 00:29:49 +1100 Subject: [PATCH 3/7] Update package requirements --- requirements.in | 4 ++-- requirements.txt | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/requirements.in b/requirements.in index f90853ffa691..3ade64c3d796 100644 --- a/requirements.in +++ b/requirements.in @@ -24,7 +24,7 @@ django-q-sentry # sentry.io integration for django-q django-sesame # Magic link authentication django-sql-utils # Advanced query annotation / aggregation django-sslserver # Secure HTTP development server -django-stdimage<6.0.0 # Advanced ImageField management # FIXED 2022-06-29 6.0.0 breaks serialization for django-q +django-stdimage # Advanced ImageField management # FIXED 2022-06-29 6.0.0 breaks serialization for django-q django-taggit # Tagging support django-user-sessions # user sessions in DB django-weasyprint # django weasyprint integration @@ -37,7 +37,7 @@ drf-spectacular # DRF API documentation feedparser # RSS newsfeed parser gunicorn # Gunicorn web server pdf2image # PDF to image conversion -pillow>=10.0.1 # Image manipulation +pillow # Image manipulation pint==0.21 # Unit conversion # FIXED 2023-05-30 breaks tests https://github.com/matmair/InvenTree/actions/runs/5095665936/jobs/9160852560 python-barcode[images] # Barcode generator python-dotenv # Environment variable management diff --git a/requirements.txt b/requirements.txt index a5176bfd0b52..da2813c97879 100644 --- a/requirements.txt +++ b/requirements.txt @@ -135,7 +135,7 @@ django-sql-utils==0.6.1 # via -r requirements.in django-sslserver==0.22 # via -r requirements.in -django-stdimage==5.3.0 +django-stdimage==6.0.1 # via -r requirements.in django-taggit==4.0.0 # via -r requirements.in @@ -201,7 +201,7 @@ packaging==23.1 # via gunicorn pdf2image==1.16.3 # via -r requirements.in -pillow==10.0.1 +pillow==10.1.0 # via # -r requirements.in # django-stdimage From d0fc2a1d9c37307dc7c52cd1dcf30f89d4cf0e50 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Thu, 19 Oct 2023 00:58:48 +1100 Subject: [PATCH 4/7] Improve offload_task function: - Return True if the task runs or was offloaded - Improved warning information --- InvenTree/InvenTree/tasks.py | 45 ++++++++++++++++++++++++++++-------- 1 file changed, 35 insertions(+), 10 deletions(-) diff --git a/InvenTree/InvenTree/tasks.py b/InvenTree/InvenTree/tasks.py index 12ad120e30f5..8f242fa26487 100644 --- a/InvenTree/InvenTree/tasks.py +++ b/InvenTree/InvenTree/tasks.py @@ -162,13 +162,17 @@ def record_task_success(task_name: str): InvenTreeSetting.set_setting(f'_{task_name}_SUCCESS', datetime.now().isoformat(), None) -def offload_task(taskname, *args, force_async=False, force_sync=False, **kwargs): +def offload_task(taskname, *args, force_async=False, force_sync=False, **kwargs) -> bool: """Create an AsyncTask if workers are running. This is different to a 'scheduled' task, in that it only runs once! - If workers are not running or force_sync flag - is set then the task is ran synchronously. + If workers are not running or force_sync flag, is set then the task is ran synchronously. + + Returns: + bool: True if the task was offloaded (or ran), False otherwise """ + import traceback + try: import importlib @@ -177,19 +181,32 @@ def offload_task(taskname, *args, force_async=False, force_sync=False, **kwargs) from InvenTree.status import is_worker_running except AppRegistryNotReady: # pragma: no cover logger.warning("Could not offload task '%s' - app registry not ready", taskname) - return + + if force_async: + # Cannot async the task, so return False + return False + else: + force_sync = True except (OperationalError, ProgrammingError): # pragma: no cover raise_warning(f"Could not offload task '{taskname}' - database not ready") + if force_async: + # Cannot async the task, so return False + return False + else: + force_sync = True + if force_async or (is_worker_running() and not force_sync): # Running as asynchronous task try: task = AsyncTask(taskname, *args, **kwargs) task.run() except ImportError: - raise_warning(f"WARNING: '{taskname}' not started - Function not found") + raise_warning(f"WARNING: '{taskname}' not offloaded - Function not found") + return False except Exception as exc: - raise_warning(f"WARNING: '{taskname}' not started due to {type(exc)}") + raise_warning(f"WARNING: '{taskname}' not offloaded due to {str(exc)}") + return False else: if callable(taskname): @@ -202,14 +219,14 @@ def offload_task(taskname, *args, force_async=False, force_sync=False, **kwargs) app_mod = app + '.' + mod except ValueError: raise_warning(f"WARNING: '{taskname}' not started - Malformed function path") - return + return False # Import module from app try: _mod = importlib.import_module(app_mod) except ModuleNotFoundError: raise_warning(f"WARNING: '{taskname}' not started - No module named '{app_mod}'") - return + return False # Retrieve function try: @@ -223,10 +240,18 @@ def offload_task(taskname, *args, force_async=False, force_sync=False, **kwargs) _func = eval(func) # pragma: no cover except NameError: raise_warning(f"WARNING: '{taskname}' not started - No function named '{func}'") - return + return False # Workers are not running: run it as synchronous task - _func(*args, **kwargs) + try: + _func(*args, **kwargs) + except Exception as exc: + raise_warning(f"WARNING: '{taskname}' not started due to {str(exc)}") + return False + + # Finally, task either completed successfully or was offloaded + return True + @dataclass() From d224668352febc4edb9ae47289e8a9970f538685 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Thu, 19 Oct 2023 01:07:56 +1100 Subject: [PATCH 5/7] Improve unit tests for task offloading - Check return value of offload_task - Check log output --- InvenTree/InvenTree/tests.py | 45 +++++++++++++++++++++++------------- requirements.in | 2 +- 2 files changed, 30 insertions(+), 17 deletions(-) diff --git a/InvenTree/InvenTree/tests.py b/InvenTree/InvenTree/tests.py index 8fec1fff2819..520ea8728cea 100644 --- a/InvenTree/InvenTree/tests.py +++ b/InvenTree/InvenTree/tests.py @@ -63,7 +63,7 @@ def test_base_units(self): self.assertAlmostEqual(float(q.magnitude), expected, 0.01) def test_dimensionless_units(self): - """Tests for 'dimensonless' unit quantities""" + """Tests for 'dimensionless' unit quantities""" # Test some dimensionless units tests = { @@ -857,7 +857,7 @@ def test_rates(self): class TestStatus(TestCase): """Unit tests for status functions.""" - def test_check_system_healt(self): + def test_check_system_health(self): """Test that the system health check is false in testing -> background worker not running.""" self.assertEqual(status.check_system_health(), False) @@ -953,7 +953,7 @@ def test_initial_install(self): InvenTreeSetting.set_setting('PLUGIN_ON_STARTUP', True, self.user) registry.reload_plugins(full_reload=True) - # Check that there was anotehr run + # Check that there was another run response = registry.install_plugin_file() self.assertEqual(response, True) @@ -1061,25 +1061,38 @@ def test_offload_tasks(self): Ref: https://github.com/inventree/InvenTree/pull/3273 """ - offload_task( - 'dummy_tasks.parts', - part=Part.objects.get(pk=1), - cat=PartCategory.objects.get(pk=1), - force_async=True - ) - - offload_task( + self.assertTrue(offload_task( 'dummy_tasks.stock', item=StockItem.objects.get(pk=1), loc=StockLocation.objects.get(pk=1), force_async=True - ) + )) - offload_task( + self.assertTrue(offload_task( 'dummy_task.numbers', 1, 2, 3, 4, 5, force_async=True - ) + )) + + # Offload a dummy task, but force sync + # This should fail, because the function does not exist + with self.assertLogs(logger='inventree', level='WARNING') as log: + self.assertFalse(offload_task( + 'dummy_task.numbers', + 1, 1, 1, + force_sync=True + )) + + self.assertIn("Malformed function path", str(log.output)) + + # Offload dummy task with a Part instance + # This should succeed, ensuring that the Part instance is correctly pickled + self.assertTrue(offload_task( + 'dummy_tasks.parts', + part=Part.objects.get(pk=1), + cat=PartCategory.objects.get(pk=1), + force_async=True + )) def test_daily_holdoff(self): """Tests for daily task holdoff helper functions""" @@ -1147,7 +1160,7 @@ def test_barcode_model_type(self): self.assertEqual(StockItem.barcode_model_type(), 'stockitem') self.assertEqual(StockLocation.barcode_model_type(), 'stocklocation') - def test_bacode_hash(self): + def test_barcode_hash(self): """Test that the barcode hashing function provides correct results""" # Test multiple values for the hashing function @@ -1176,7 +1189,7 @@ def test_svg_sanitizer(self): # Test that valid string self.assertEqual(valid_string, sanitize_svg(valid_string)) - # Test that invalid string is cleanded + # Test that invalid string is cleaned self.assertNotEqual(dangerous_string, sanitize_svg(dangerous_string)) diff --git a/requirements.in b/requirements.in index 3ade64c3d796..92dab2f8128e 100644 --- a/requirements.in +++ b/requirements.in @@ -24,7 +24,7 @@ django-q-sentry # sentry.io integration for django-q django-sesame # Magic link authentication django-sql-utils # Advanced query annotation / aggregation django-sslserver # Secure HTTP development server -django-stdimage # Advanced ImageField management # FIXED 2022-06-29 6.0.0 breaks serialization for django-q +django-stdimage # Advanced ImageField management django-taggit # Tagging support django-user-sessions # user sessions in DB django-weasyprint # django weasyprint integration From c0015f3428560f7a941e3b2b5bcb0f4416ac6deb Mon Sep 17 00:00:00 2001 From: Oliver Date: Tue, 7 Nov 2023 09:39:10 +1100 Subject: [PATCH 6/7] Bump django-stdimage --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 54a847a89fab..6bfb26ec62ea 100644 --- a/requirements.txt +++ b/requirements.txt @@ -135,7 +135,7 @@ django-sql-utils==0.7.0 # via -r requirements.in django-sslserver==0.22 # via -r requirements.in -django-stdimage==6.0.1 +django-stdimage==6.0.2 # via -r requirements.in django-taggit==4.0.0 # via -r requirements.in From c77b398c060701b57d48a39d0381d769b2297c1a Mon Sep 17 00:00:00 2001 From: Oliver Date: Tue, 7 Nov 2023 09:41:46 +1100 Subject: [PATCH 7/7] Fix tasks.py --- InvenTree/InvenTree/tasks.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/InvenTree/InvenTree/tasks.py b/InvenTree/InvenTree/tasks.py index e22ab307fb9f..6dbcb85bbf36 100644 --- a/InvenTree/InvenTree/tasks.py +++ b/InvenTree/InvenTree/tasks.py @@ -175,7 +175,7 @@ def offload_task(taskname, *args, force_async=False, force_sync=False, **kwargs) from InvenTree.status import is_worker_running except AppRegistryNotReady: # pragma: no cover logger.warning("Could not offload task '%s' - app registry not ready", taskname) - + if force_async: # Cannot async the task, so return False return False @@ -242,12 +242,11 @@ def offload_task(taskname, *args, force_async=False, force_sync=False, **kwargs) except Exception as exc: raise_warning(f"WARNING: '{taskname}' not started due to {str(exc)}") return False - + # Finally, task either completed successfully or was offloaded return True - @dataclass() class ScheduledTask: """A scheduled task.