From 624711c405505c409fe4622131a76241df2ff07a Mon Sep 17 00:00:00 2001 From: Andrew Sutulov Date: Thu, 30 May 2024 17:19:42 +0200 Subject: [PATCH] add Temporal.io related code encapsulation --- CHANGELOG.md | 4 ++ README.md | 10 ++++- dev/temporalio/__init__.py | 0 dev/{ => temporalio}/activities.py | 2 +- dev/temporalio/foo/__init__.py | 0 dev/temporalio/foo/bar/__init__.py | 0 dev/temporalio/foo/bar/workflows_bar.py | 0 dev/temporalio/foo/foo_workflows.py | 0 dev/temporalio/foo/invalid_workflows.txt | 0 dev/{temporalio.py => temporalio/queues.py} | 0 dev/{ => temporalio}/schedules.py | 2 +- dev/{ => temporalio}/workflows.py | 2 +- dev/tests/registry/test_queue_activities.py | 11 +++-- dev/tests/registry/test_queue_workflows.py | 9 ++-- dev/tests/registry/test_schedules.py | 9 ++-- dev/tests/test_settings.py | 6 +++ dev/tests/utils/__init__.py | 0 dev/tests/utils/test_autodiscover_modules.py | 32 ++++++++++++++ django_temporalio/conf.py | 5 +++ django_temporalio/registry.py | 9 ++-- django_temporalio/utils.py | 46 ++++++++++++++++++++ 21 files changed, 129 insertions(+), 18 deletions(-) create mode 100644 dev/temporalio/__init__.py rename dev/{ => temporalio}/activities.py (79%) create mode 100644 dev/temporalio/foo/__init__.py create mode 100644 dev/temporalio/foo/bar/__init__.py create mode 100644 dev/temporalio/foo/bar/workflows_bar.py create mode 100644 dev/temporalio/foo/foo_workflows.py create mode 100644 dev/temporalio/foo/invalid_workflows.txt rename dev/{temporalio.py => temporalio/queues.py} (100%) rename dev/{ => temporalio}/schedules.py (93%) rename dev/{ => temporalio}/workflows.py (82%) create mode 100644 dev/tests/utils/__init__.py create mode 100644 dev/tests/utils/test_autodiscover_modules.py create mode 100644 django_temporalio/utils.py diff --git a/CHANGELOG.md b/CHANGELOG.md index c4a5cab..02728a8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ ## 1.0.0 (2024-05-16) * Initial release + +## 1.1.0 (2024-05-30) + +* add Temporal.io related code encapsulation diff --git a/README.md b/README.md index b0065fa..bbda0d1 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,7 @@ from temporalio.worker import WorkerConfig DJANGO_TEMPORALIO = { "URL": "localhost:7233", + "BASE_MODULE": "path.to.module", "WORKER_CONFIGS": { "main": WorkerConfig( task_queue="MAIN_TASK_QUEUE", @@ -45,12 +46,18 @@ DJANGO_TEMPORALIO = { ## Usage +Activities, workflows and schedules should be placed inside the base module defined by the `BASE_MODULE` setting, +preferably outside of any Django application, in order to keep the uses of +the [imports_passed_through](https://python.temporal.io/temporalio.workflow.unsafe.html) context manager encapsulated +inside the module, along with Temporal.io related code. + ### Workflow and Activity Registry The registry is a singleton that holds mappings between queue names and registered activities and workflows. You can register activities and workflows using the `register` method. -Activities and workflows should be declared in `workflows.py` and `activities.py` modules respectively. +Activities and workflows should be declared in modules matching the following patterns `*workflows*.py` and +`*activities*.py` respectively. ```python from temporalio import activity, workflow @@ -123,3 +130,4 @@ DJANGO_TEMPORALIO: A dictionary containing the following keys: - NAMESPACE: The Temporal.io namespace to use, defaults to `default` - WORKER_CONFIGS: A dictionary containing worker configurations. The key is the worker name and the value is a `WorkerConfig` instance. +- BASE_MODULE: A python module that holds workflows, activities and schedules, defaults to `None` diff --git a/dev/temporalio/__init__.py b/dev/temporalio/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dev/activities.py b/dev/temporalio/activities.py similarity index 79% rename from dev/activities.py rename to dev/temporalio/activities.py index 450e081..1d7ddb7 100644 --- a/dev/activities.py +++ b/dev/temporalio/activities.py @@ -1,6 +1,6 @@ from temporalio import activity -from dev.temporalio import TestTaskQueues +from dev.temporalio.queues import TestTaskQueues from django_temporalio.registry import queue_activities diff --git a/dev/temporalio/foo/__init__.py b/dev/temporalio/foo/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dev/temporalio/foo/bar/__init__.py b/dev/temporalio/foo/bar/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dev/temporalio/foo/bar/workflows_bar.py b/dev/temporalio/foo/bar/workflows_bar.py new file mode 100644 index 0000000..e69de29 diff --git a/dev/temporalio/foo/foo_workflows.py b/dev/temporalio/foo/foo_workflows.py new file mode 100644 index 0000000..e69de29 diff --git a/dev/temporalio/foo/invalid_workflows.txt b/dev/temporalio/foo/invalid_workflows.txt new file mode 100644 index 0000000..e69de29 diff --git a/dev/temporalio.py b/dev/temporalio/queues.py similarity index 100% rename from dev/temporalio.py rename to dev/temporalio/queues.py diff --git a/dev/schedules.py b/dev/temporalio/schedules.py similarity index 93% rename from dev/schedules.py rename to dev/temporalio/schedules.py index 7986358..396487e 100644 --- a/dev/schedules.py +++ b/dev/temporalio/schedules.py @@ -6,7 +6,7 @@ ScheduleSpec, ) -from dev.temporalio import TestTaskQueues +from dev.temporalio.queues import TestTaskQueues from django_temporalio.registry import schedules schedules.register( diff --git a/dev/workflows.py b/dev/temporalio/workflows.py similarity index 82% rename from dev/workflows.py rename to dev/temporalio/workflows.py index fb63c6a..7cbe02d 100644 --- a/dev/workflows.py +++ b/dev/temporalio/workflows.py @@ -1,6 +1,6 @@ from temporalio import workflow -from dev.temporalio import TestTaskQueues +from dev.temporalio.queues import TestTaskQueues from django_temporalio.registry import queue_workflows diff --git a/dev/tests/registry/test_queue_activities.py b/dev/tests/registry/test_queue_activities.py index e3b9a9a..34f70da 100644 --- a/dev/tests/registry/test_queue_activities.py +++ b/dev/tests/registry/test_queue_activities.py @@ -1,10 +1,12 @@ from unittest import TestCase, mock -from django.utils.module_loading import autodiscover_modules +from django.test import override_settings from temporalio import activity -from dev.temporalio import TestTaskQueues +from dev.temporalio.queues import TestTaskQueues +from django_temporalio.conf import SETTINGS_KEY from django_temporalio.registry import queue_activities +from django_temporalio.utils import autodiscover_modules @activity.defn @@ -20,6 +22,7 @@ class QueueActivityRegistryTestCase(TestCase): def tearDown(self): queue_activities.clear_registry() + @override_settings(**{SETTINGS_KEY: {"BASE_MODULE": "dev.temporalio"}}) @mock.patch( "django_temporalio.registry.autodiscover_modules", wraps=autodiscover_modules ) @@ -34,14 +37,14 @@ def test_get_registry(self, mock_register, mock_autodiscover_modules): registry = queue_activities.get_registry() mock_register.assert_called_once_with(TestTaskQueues.MAIN) - mock_autodiscover_modules.assert_called_once_with("activities") + mock_autodiscover_modules.assert_called_once_with("*activities*") self.assertEqual(len(registry), 1) self.assertIn(TestTaskQueues.MAIN, registry) activities = registry[TestTaskQueues.MAIN] self.assertEqual(len(activities), 1) self.assertEqual( f"{activities[0].__module__}.{activities[0].__name__}", - "dev.activities.test_activity", + "dev.temporalio.activities.test_activity", ) @mock.patch("django_temporalio.registry.autodiscover_modules") diff --git a/dev/tests/registry/test_queue_workflows.py b/dev/tests/registry/test_queue_workflows.py index a87abca..7eb1269 100644 --- a/dev/tests/registry/test_queue_workflows.py +++ b/dev/tests/registry/test_queue_workflows.py @@ -1,8 +1,10 @@ from unittest import TestCase, mock +from django.test import override_settings from temporalio import workflow -from dev.temporalio import TestTaskQueues +from dev.temporalio.queues import TestTaskQueues +from django_temporalio.conf import SETTINGS_KEY from django_temporalio.registry import queue_workflows, autodiscover_modules @@ -21,6 +23,7 @@ class QueueWorkflowRegistryTestCase(TestCase): def tearDown(self): queue_workflows.clear_registry() + @override_settings(**{SETTINGS_KEY: {"BASE_MODULE": "dev.temporalio"}}) @mock.patch( "django_temporalio.registry.autodiscover_modules", wraps=autodiscover_modules ) @@ -35,13 +38,13 @@ def test_get_registry(self, mock_register, mock_autodiscover_modules): registry = queue_workflows.get_registry() mock_register.assert_called_once_with(TestTaskQueues.MAIN) - mock_autodiscover_modules.assert_called_once_with("workflows") + mock_autodiscover_modules.assert_called_once_with("*workflows*") self.assertEqual(len(registry), 1) self.assertIn(TestTaskQueues.MAIN, registry) workflows = registry[TestTaskQueues.MAIN] self.assertEqual(len(workflows), 1) self.assertEqual( - "dev.workflows.TestWorkflow", + "dev.temporalio.workflows.TestWorkflow", f"{workflows[0].__module__}.{workflows[0].__name__}", ) diff --git a/dev/tests/registry/test_schedules.py b/dev/tests/registry/test_schedules.py index 44d10bb..3106f0c 100644 --- a/dev/tests/registry/test_schedules.py +++ b/dev/tests/registry/test_schedules.py @@ -1,6 +1,6 @@ from unittest import TestCase, mock -from django.utils.module_loading import autodiscover_modules +from django.test import override_settings from temporalio.client import ( ScheduleActionStartWorkflow, Schedule, @@ -9,8 +9,10 @@ ScheduleRange, ) -from dev.temporalio import TestTaskQueues +from dev.temporalio.queues import TestTaskQueues +from django_temporalio.conf import SETTINGS_KEY from django_temporalio.registry import schedules +from django_temporalio.utils import autodiscover_modules class ScheduleRegistryTestCase(TestCase): @@ -41,6 +43,7 @@ def setUpClass(cls): def tearDown(self): schedules.clear_registry() + @override_settings(**{SETTINGS_KEY: {"BASE_MODULE": "dev.temporalio"}}) @mock.patch( "django_temporalio.registry.autodiscover_modules", wraps=autodiscover_modules ) @@ -52,7 +55,7 @@ def test_get_registry(self, mock_register, mock_autodiscover_modules): registry = schedules.get_registry() mock_register.assert_called_once() - mock_autodiscover_modules.assert_called_once_with("schedules") + mock_autodiscover_modules.assert_called_once_with("*schedules*") self.assertEqual(len(registry), 1) self.assertIn("do-cool-stuff-every-hour", registry) diff --git a/dev/tests/test_settings.py b/dev/tests/test_settings.py index df03f9c..a0060c8 100644 --- a/dev/tests/test_settings.py +++ b/dev/tests/test_settings.py @@ -20,12 +20,14 @@ def test_default_settings(self): self.assertEqual(temporalio_settings.URL, DEFAULTS["URL"]) self.assertEqual(temporalio_settings.NAMESPACE, DEFAULTS["NAMESPACE"]) self.assertEqual(temporalio_settings.WORKER_CONFIGS, DEFAULTS["WORKER_CONFIGS"]) + self.assertEqual(temporalio_settings.BASE_MODULE, DEFAULTS["BASE_MODULE"]) def test_user_settings(self): user_settings = { "URL": "http://temporal:7233", "NAMESPACE": "main", "WORKER_CONFIGS": {"main": "config"}, + "BASE_MODULE": "dev.temporalio", } with override_settings(**{SETTINGS_KEY: user_settings}): self.assertEqual(temporalio_settings.URL, user_settings["URL"]) @@ -33,6 +35,9 @@ def test_user_settings(self): self.assertEqual( temporalio_settings.WORKER_CONFIGS, user_settings["WORKER_CONFIGS"] ) + self.assertEqual( + temporalio_settings.BASE_MODULE, user_settings["BASE_MODULE"] + ) def test_fallback_to_defaults(self): user_settings = { @@ -44,6 +49,7 @@ def test_fallback_to_defaults(self): self.assertEqual( temporalio_settings.WORKER_CONFIGS, DEFAULTS["WORKER_CONFIGS"] ) + self.assertEqual(temporalio_settings.BASE_MODULE, DEFAULTS["BASE_MODULE"]) def test_invalid_setting(self): with self.assertRaises(AttributeError): diff --git a/dev/tests/utils/__init__.py b/dev/tests/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dev/tests/utils/test_autodiscover_modules.py b/dev/tests/utils/test_autodiscover_modules.py new file mode 100644 index 0000000..0a0c075 --- /dev/null +++ b/dev/tests/utils/test_autodiscover_modules.py @@ -0,0 +1,32 @@ +from importlib import import_module +from unittest import TestCase, mock + +from django.test import override_settings + +from django_temporalio.conf import settings, SETTINGS_KEY, SettingIsNotSetError +from django_temporalio.utils import autodiscover_modules + + +class AutodiscoverModulesTestCase(TestCase): + """ + Test case for utils.autodiscover_modules. + """ + + @override_settings(**{SETTINGS_KEY: {"BASE_MODULE": "dev.temporalio"}}) + @mock.patch("django_temporalio.utils.import_module", wraps=import_module) + def test_autodiscover_modules(self, import_module_mock): + autodiscover_modules("*workflows*") + + import_module_mock.assert_has_calls( + [ + mock.call("dev.temporalio"), + mock.call("dev.temporalio.workflows"), + mock.call("dev.temporalio.foo.foo_workflows"), + mock.call("dev.temporalio.foo.bar.workflows_bar"), + ] + ) + + def test_autodiscover_modules_raises_exception(self): + self.assertIsNone(settings.BASE_MODULE) + with self.assertRaises(SettingIsNotSetError): + autodiscover_modules("*workflows*") diff --git a/django_temporalio/conf.py b/django_temporalio/conf.py index 09e6aeb..d566e71 100644 --- a/django_temporalio/conf.py +++ b/django_temporalio/conf.py @@ -20,9 +20,14 @@ "URL": "http://localhost:7233", "NAMESPACE": "default", "WORKER_CONFIGS": {}, + "BASE_MODULE": None, } +class SettingIsNotSetError(Exception): + pass + + class Settings: def __init__(self): self.defaults = DEFAULTS diff --git a/django_temporalio/registry.py b/django_temporalio/registry.py index 32e949d..3cb6fb8 100644 --- a/django_temporalio/registry.py +++ b/django_temporalio/registry.py @@ -2,9 +2,10 @@ from functools import wraps from typing import Callable, Sequence, Type -from django.utils.module_loading import autodiscover_modules from temporalio.client import Schedule +from django_temporalio.utils import autodiscover_modules + class ScheduleRegistry: _registry: dict[str, Schedule] @@ -26,7 +27,7 @@ def register(self, schedule_id: str, schedule: Schedule): self._registry[schedule_id] = schedule def get_registry(self): - autodiscover_modules("schedules") + autodiscover_modules("*schedules*") return self._registry def clear_registry(self): @@ -85,8 +86,8 @@ def get_registry(self): schedules = ScheduleRegistry() -queue_workflows = QueueRegistry("workflows", "__temporal_workflow_definition") -queue_activities = QueueRegistry("activities", "__temporal_activity_definition") +queue_workflows = QueueRegistry("*workflows*", "__temporal_workflow_definition") +queue_activities = QueueRegistry("*activities*", "__temporal_activity_definition") @dataclass diff --git a/django_temporalio/utils.py b/django_temporalio/utils.py new file mode 100644 index 0000000..f2b2647 --- /dev/null +++ b/django_temporalio/utils.py @@ -0,0 +1,46 @@ +import fnmatch +import os +from importlib import import_module + +from django_temporalio.conf import settings, SettingIsNotSetError + + +def autodiscover_modules(related_name_pattern: str): + """ + Autodiscover modules matching the related name pattern in the base module. + + Example for the following directory structure: + + foo/ + workflows.py + activities.py + bar/ + bar_workflows.py + activities.py + baz/ + workflows_baz.py + activities.py + + Calling `autodiscover_modules('foo', '*workflows*')` will discover the following modules: + - foo.workflows + - foo.bar.bar_workflows + - foo.bar.baz.workflows_baz + """ + base_module_name = settings.BASE_MODULE + + if not base_module_name: + raise SettingIsNotSetError("BASE_MODULE setting must be set.") + + base_module = import_module(base_module_name) + base_module_path = base_module.__path__[0] + + for root, _, files in os.walk(base_module_path): + for file in files: + if not fnmatch.fnmatch(file, f"{related_name_pattern}.py"): + continue + + module_name = root.replace(base_module_path, base_module_name).replace( + os.sep, "." + ) + # import pdb; pdb.set_trace() + import_module(f"{module_name}.{file[:-3]}")