From 13b31804b59a6b52c24aae158c815e9538c24ebe Mon Sep 17 00:00:00 2001 From: goFrendiAsgard Date: Thu, 19 Oct 2023 20:27:12 +0700 Subject: [PATCH 01/18] update version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 1fabd7b4..9ba5374e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "flit_core.buildapi" [project] name = "zrb" -version = "0.0.105" +version = "0.0.106" authors = [ { name="Go Frendi Gunawan", email="gofrendiasgard@gmail.com" }, ] From 8aff276559fc5e3f67ad44248a852672ae002c82 Mon Sep 17 00:00:00 2001 From: goFrendiAsgard Date: Sun, 5 Nov 2023 14:29:25 +0700 Subject: [PATCH 02/18] update pyproject --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 954fe75f..24451fe5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "flit_core.buildapi" [project] name = "zrb" -version = "0.0.109" +version = "0.0.110" authors = [ { name="Go Frendi Gunawan", email="gofrendiasgard@gmail.com" }, ] From 3e3070e51ffbb92a107e73ae3a2b77934b60b242 Mon Sep 17 00:00:00 2001 From: goFrendiAsgard Date: Sun, 5 Nov 2023 14:42:24 +0700 Subject: [PATCH 03/18] update docs --- ...-same-variable-to-define-different-task.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/docs/oops-i-did-it-again/using-the-same-variable-to-define-different-task.md b/docs/oops-i-did-it-again/using-the-same-variable-to-define-different-task.md index 3064e8aa..5fb40880 100644 --- a/docs/oops-i-did-it-again/using-the-same-variable-to-define-different-task.md +++ b/docs/oops-i-did-it-again/using-the-same-variable-to-define-different-task.md @@ -5,7 +5,26 @@ ```python from zrb import runner, CmdTask +prepare = CmdTask( + name='prepare-python-project', + cmd='pip install -R requirements.txt' +) +runner.register(prepare) +prepare = CmdTask( + name='prepare-node-project', + cmd='npm install' +) +runner.register(prepare) + +run_fastapi = CmdTask( + name='run-fastapi', + cmd='uvicon main:app' + upstreams=[ + prepare, # <-- Here is the problem, `npm install`` or `pip install`? + ] +) +runner.register(run_fastapi) ``` 🔖 [Table of Contents](../README.md) / [Oops, I Did It Again](README.md) From 1828fe7c0aea6d8a8630efcc8b7220194bfed4e0 Mon Sep 17 00:00:00 2001 From: goFrendiAsgard Date: Sun, 5 Nov 2023 16:06:34 +0700 Subject: [PATCH 04/18] update docs --- .../using-the-same-variable-to-define-different-task.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/oops-i-did-it-again/using-the-same-variable-to-define-different-task.md b/docs/oops-i-did-it-again/using-the-same-variable-to-define-different-task.md index 5fb40880..6762c52e 100644 --- a/docs/oops-i-did-it-again/using-the-same-variable-to-define-different-task.md +++ b/docs/oops-i-did-it-again/using-the-same-variable-to-define-different-task.md @@ -27,4 +27,13 @@ run_fastapi = CmdTask( runner.register(run_fastapi) ``` +You can see that `prepare-python-project` and `prepare-node-project` are assigned to the same variable. + +Using that variable as upstream or checker will lead to a tricky situation. In our case, we want to perform `pip install` before starting Fast API. But since we re-assign the variable to `prepare-node-project`, we will got `npm install` instead. + +# Avoiding the Problem + +Beware of your variable name. Give your variable the same name as your task name. + + 🔖 [Table of Contents](../README.md) / [Oops, I Did It Again](README.md) From 66fb07732de6944b91ca1358ce8efa84df9787e0 Mon Sep 17 00:00:00 2001 From: goFrendiAsgard Date: Mon, 6 Nov 2023 08:08:46 +0700 Subject: [PATCH 05/18] add names --- src/zrb/helper/accessories/name.py | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/src/zrb/helper/accessories/name.py b/src/zrb/helper/accessories/name.py index 07b5100b..bbc7d46b 100644 --- a/src/zrb/helper/accessories/name.py +++ b/src/zrb/helper/accessories/name.py @@ -10,19 +10,23 @@ def get_random_name( digit_count: int = 4 ) -> str: prefixes = [ - "aurum", "argentum", "platinum", "mercurius", "sulfur", "sal", - "luna", "sol", "ferrum", "cuprum", "argent", "aurora", "citrin", - "coral", "diamond", "dragon", "emerald", "garnet", "jade", "onyx", - "opal", "pearl", "ruby", "sapphire", "topaz", "turquoise", "verde", - "zircon" + 'aurum', 'argentum', 'platinum', 'mercurius', 'sulfur', 'sal', + 'luna', 'sol', 'ferrum', 'cuprum', 'argent', 'aurora', 'citrin', + 'coral', 'diamond', 'dragon', 'emerald', 'garnet', 'jade', 'onyx', + 'opal', 'pearl', 'ruby', 'sapphire', 'topaz', 'turquoise', 'verde', + 'zircon', 'venerable', 'serene', 'temptuous', 'inferna', 'celestial', + 'ancient', 'ethereal', 'occult', 'primal', 'arcane', 'mystic', 'dark', + 'divine' ] suffixes = [ - "philosophorum", "spiritus", "tinctura", "essentia", "elixir", - "praeparatum", "aether", "vitae", "lapis", "metallum", "aureum", - "caelestis", "chrysopoeia", "cosmicum", "deum", "draconis", - "elementorum", "hermetica", "illuminationis", "magnum", "mysticum", - "occultum", "omnipotentis", "philosophia", "praestantissimum", - "quintessentia", "regeneratio", "universalis" + 'philosophorum', 'spiritus', 'tinctura', 'essentia', 'elixir', + 'praeparatum', 'aether', 'vitae', 'lapis', 'metallum', 'aureum', + 'caelestis', 'chrysopoeia', 'cosmicum', 'deum', 'draconis', + 'elementorum', 'hermetica', 'illuminationis', 'magnum', 'mysticum', + 'occultum', 'omnipotentis', 'philosophia', 'praestantissimum', + 'quintessentia', 'regeneratio', 'universalis', 'alchemia', + 'transmutatio', 'spiritualis', 'potentia', 'siderium', 'philogia', + 'occulta', 'perpetua', 'infinitum', 'animus' ] prefix = random.choice(prefixes) suffix = random.choice(suffixes) From bf5e64cd710377d58b9bad1fe25e37bfccf33b37 Mon Sep 17 00:00:00 2001 From: goFrendiAsgard Date: Mon, 6 Nov 2023 08:17:00 +0700 Subject: [PATCH 06/18] make execution id more random --- src/zrb/task/base_task.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/zrb/task/base_task.py b/src/zrb/task/base_task.py index 72fc9598..169d9c39 100644 --- a/src/zrb/task/base_task.py +++ b/src/zrb/task/base_task.py @@ -310,7 +310,9 @@ async def _run_and_check_all( try: self._start_timer() if self.get_execution_id() == '': - self.set_execution_id(get_random_name()) + self.set_execution_id( + get_random_name(add_random_digit=True, digit_count=5) + ) self.log_info('Set input and env map') await self._set_keyval(kwargs=kwargs, env_prefix=env_prefix) self.log_info('Set run kwargs') From 29a1445b5d9dedcd3d7e9938beccb2dc32fb863f Mon Sep 17 00:00:00 2001 From: goFrendiAsgard Date: Mon, 6 Nov 2023 18:35:19 +0700 Subject: [PATCH 07/18] improve name --- src/zrb/builtin/git.py | 4 +--- src/zrb/helper/accessories/name.py | 36 +++++++++++++++++------------- 2 files changed, 22 insertions(+), 18 deletions(-) diff --git a/src/zrb/builtin/git.py b/src/zrb/builtin/git.py index ae11eed1..0b83395d 100644 --- a/src/zrb/builtin/git.py +++ b/src/zrb/builtin/git.py @@ -1,7 +1,6 @@ from zrb.helper.typing import Any from zrb.builtin.group import git_group from zrb.task.decorator import python_task -from zrb.task.task import Task from zrb.task_input.str_input import StrInput from zrb.task_input.bool_input import BoolInput from zrb.runner import runner @@ -26,7 +25,7 @@ prompt='Commit hash/Tag', default='HEAD' ), - BoolInput( + BoolInput( name='include-new', description='include new files', prompt='Include new files', @@ -52,7 +51,6 @@ async def get_file_changes(*args: Any, **kwargs: Any): include_new = kwargs.get('include_new', True) include_removed = kwargs.get('include_removed', True) include_updated = kwargs.get('include_updated', True) - task: Task = kwargs['_task'] modified_file_states = get_modified_file_states(commit) modified_file_keys = [] output = [] diff --git a/src/zrb/helper/accessories/name.py b/src/zrb/helper/accessories/name.py index bbc7d46b..96b8b86b 100644 --- a/src/zrb/helper/accessories/name.py +++ b/src/zrb/helper/accessories/name.py @@ -10,23 +10,29 @@ def get_random_name( digit_count: int = 4 ) -> str: prefixes = [ - 'aurum', 'argentum', 'platinum', 'mercurius', 'sulfur', 'sal', - 'luna', 'sol', 'ferrum', 'cuprum', 'argent', 'aurora', 'citrin', - 'coral', 'diamond', 'dragon', 'emerald', 'garnet', 'jade', 'onyx', - 'opal', 'pearl', 'ruby', 'sapphire', 'topaz', 'turquoise', 'verde', - 'zircon', 'venerable', 'serene', 'temptuous', 'inferna', 'celestial', - 'ancient', 'ethereal', 'occult', 'primal', 'arcane', 'mystic', 'dark', - 'divine' + 'albedo', 'argent', 'argentum', 'aurora', 'aurum', 'azure', + 'basilisk', 'cerulean', 'chimeric', 'citrin', 'coral', 'crimson', + 'diamond', 'draco', 'dragon', 'emerald', 'ethereal', 'ferrum', + 'flammeus', 'garnet', 'glacial', 'glimmering', 'glistening', 'golden', + 'helios', 'igneous', 'imperial', 'jade', 'luminous', 'luna', 'lunar', + 'mystic', 'nephrite', 'nocturnal', 'obsidian', 'opal', 'pearl', + 'platinum', 'prismatic', 'ruby', 'sapphire', 'serpentine', 'silver', + 'sol', 'solar', 'spiritual', 'stellar', 'tempest', 'topaz', + 'turquoise', 'verde', 'vermillion', 'vitreous', 'zephyr', 'zircon' ] suffixes = [ - 'philosophorum', 'spiritus', 'tinctura', 'essentia', 'elixir', - 'praeparatum', 'aether', 'vitae', 'lapis', 'metallum', 'aureum', - 'caelestis', 'chrysopoeia', 'cosmicum', 'deum', 'draconis', - 'elementorum', 'hermetica', 'illuminationis', 'magnum', 'mysticum', - 'occultum', 'omnipotentis', 'philosophia', 'praestantissimum', - 'quintessentia', 'regeneratio', 'universalis', 'alchemia', - 'transmutatio', 'spiritualis', 'potentia', 'siderium', 'philogia', - 'occulta', 'perpetua', 'infinitum', 'animus' + 'aether', 'albedo', 'alchemy', 'arcana', 'aureum', 'aetheris', + 'anima', 'astralis', 'caelestis', 'chrysopoeia', 'cosmicum', + 'crystallum', 'deum', 'divinitas', 'draconis', 'elementorum', 'elixir', + 'essentia', 'eternis', 'ethereus', 'fatum', 'flamma', 'fulgur', + 'hermetica', 'ignis', 'illuminationis', 'imperium', 'incantatum', + 'infinitum', 'lapis', 'lux', 'magicae', 'magnum', 'materia', + 'metallum', 'mysticum', 'natura', 'occultum', 'omnipotentis', + 'opulentia', 'philosophia', 'philosophorum', 'praeparatum', + 'praestantissimum', 'prima', 'primordium', 'quintessentia', + 'regeneratio', 'ritualis', 'sanctum', 'spiritus', 'tenebris', + 'terra', 'tinctura', 'transmutationis', 'universalis', 'vapores', + 'venenum', 'veritas', 'vitae', 'volatus' ] prefix = random.choice(prefixes) suffix = random.choice(suffixes) From f78bbeb3e5746def4945b0d34272fd3edbf78412 Mon Sep 17 00:00:00 2001 From: goFrendiAsgard Date: Mon, 6 Nov 2023 23:10:32 +0000 Subject: [PATCH 08/18] add config map --- src/zrb/task/base_remote_cmd_task.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/zrb/task/base_remote_cmd_task.py b/src/zrb/task/base_remote_cmd_task.py index f1e864ed..bb03553b 100644 --- a/src/zrb/task/base_remote_cmd_task.py +++ b/src/zrb/task/base_remote_cmd_task.py @@ -34,13 +34,15 @@ def __init__( user: str = '', password: str = '', ssh_key: str = '', - port: int = 22 + port: int = 22, + config_map: Optional[Mapping[str, str]] = None ): self.host = host self.user = user self.password = password self.ssh_key = ssh_key self.port = port + self.config_map = {} if config_map is None else config_map @typechecked From 9bfdc1fd85c74a257f4ab8ee4eb789f58a34f4a3 Mon Sep 17 00:00:00 2001 From: goFrendiAsgard Date: Tue, 7 Nov 2023 08:33:30 +0700 Subject: [PATCH 09/18] add --makepath to rscync --- docs/concepts/tasks/rsync-task.md | 30 +++++++++++++++++++--------- src/zrb/shell-scripts/rsync-util.sh | 6 +++--- src/zrb/task/base_remote_cmd_task.py | 5 +++++ 3 files changed, 29 insertions(+), 12 deletions(-) diff --git a/docs/concepts/tasks/rsync-task.md b/docs/concepts/tasks/rsync-task.md index 499fe04a..5eef1342 100644 --- a/docs/concepts/tasks/rsync-task.md +++ b/docs/concepts/tasks/rsync-task.md @@ -1,13 +1,13 @@ 🔖 [Table of Contents](../../README.md) / [Concepts](../README.md) / [Tasks](README.md) -# RSyncTask +# RsyncTask ```python from zrb import ( - runner, CmdTask, RSyncTask, RemoteConfig, PasswordInput, StrInput + runner, CmdTask, RsyncTask, RemoteConfig, PasswordInput, StrInput ) -upload = RSyncTask( +upload = RsyncTask( name='upload', inputs=[ PasswordInput(name='passsword'), @@ -18,17 +18,20 @@ upload = RSyncTask( RemoteConfig( host='192.168.1.10, user='ubuntu, - password='{{input.password}}' + password='{{input.password}}', + config_map={ + 'dir': '192-168-1-10' + } ) ], is_remote_src=False, - is_remote_dst=True - src='{{input.src}}', + src='$_CONFIG_MAP_DIR/{{input.src}}', + is_remote_dst=True, dst='{{input.dst}}', ) runner.register(upload) -download = RSyncTask( +download = RsyncTask( name='download', inputs=[ PasswordInput(name='passsword'), @@ -43,11 +46,20 @@ download = RSyncTask( ) ], is_remote_src=True, - is_remote_dst=False src='{{input.src}}', - dst='{{input.dst}}', + is_remote_dst=False, + dst='$_CONFIG_MAP_DIR/{{input.dst}}', ) runner.register(download) ``` +RsyncTask exposes several environments that you can use on your `src` and `dst` + +- `_CONFIG_HOST` +- `_CONFIG_PORT` +- `_CONFIG_SSH_KEY` +- `_CONFIG_USER` +- `_CONFIG_PASSWORD` +- `_CONFIG_MAP_` + 🔖 [Table of Contents](../../README.md) / [Concepts](../README.md) / [Tasks](README.md) diff --git a/src/zrb/shell-scripts/rsync-util.sh b/src/zrb/shell-scripts/rsync-util.sh index f050ba10..9f847fc3 100644 --- a/src/zrb/shell-scripts/rsync-util.sh +++ b/src/zrb/shell-scripts/rsync-util.sh @@ -2,11 +2,11 @@ set -e auth_rsync(){ if [ "$_CONFIG_SSH_KEY" != "" ] then - rsync -avz -e "ssh -i $_CONFIG_SSH_KEY -p $_CONFIG_PORT" $@ + rsync --mkpath -avz -e "ssh -i $_CONFIG_SSH_KEY -p $_CONFIG_PORT" $@ elif [ "$_CONFIG_PASSWORD" != "" ] then - sshpass -p "$_CONFIG_PASSWORD" rsync -avz -e "ssh -p $_CONFIG_PORT" $@ + sshpass -p "$_CONFIG_PASSWORD" rsync --mkpath -avz -e "ssh -p $_CONFIG_PORT" $@ else - rsync -avz -e "ssh -p $_CONFIG_PORT" $@ + rsync --mkpath -avz -e "ssh -p $_CONFIG_PORT" $@ fi } \ No newline at end of file diff --git a/src/zrb/task/base_remote_cmd_task.py b/src/zrb/task/base_remote_cmd_task.py index bb03553b..eba8487d 100644 --- a/src/zrb/task/base_remote_cmd_task.py +++ b/src/zrb/task/base_remote_cmd_task.py @@ -2,6 +2,7 @@ Any, Callable, Iterable, Mapping, Optional, Union, TypeVar ) from zrb.helper.typecheck import typechecked +from zrb.helper.util import to_snake_case from zrb.task.any_task import AnyTask from zrb.task.any_task_event_handler import ( OnTriggered, OnWaiting, OnSkipped, OnStarted, OnReady, OnRetry, OnFailed @@ -138,6 +139,10 @@ def _get_shell_env_map(self) -> Mapping[str, Any]: env_map['_CONFIG_PASSWORD'] = self.render_str( self._remote_config.password ) + for key, val in self._remote_config.config_map.items(): + upper_snake_key = to_snake_case(key).upper() + rendered_val = self.render_str(val) + env_map['_CONFIG_MAP_' + upper_snake_key] = rendered_val return env_map def _get_cmd_str(self, *args: Any, **kwargs: Any) -> str: From 7c6684be23688bf8d9efb3310d7798a82aaa53bb Mon Sep 17 00:00:00 2001 From: goFrendiAsgard Date: Tue, 7 Nov 2023 08:36:00 +0700 Subject: [PATCH 10/18] update docs --- docs/concepts/tasks/README.md | 2 +- docs/concepts/tasks/remote-cmd-task.md | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/docs/concepts/tasks/README.md b/docs/concepts/tasks/README.md index ab9259d5..87a21cd3 100644 --- a/docs/concepts/tasks/README.md +++ b/docs/concepts/tasks/README.md @@ -12,7 +12,7 @@ There are many task types in Zrb. Every task has its own specific use cases: - [ResourceMaker](resource-maker.md): Generate artifacts/resources based on templates - [FlowTask](flow-task.md): Put `CmdTask` and `python task` into single flow. - [RemoteCmdTask](remote-cmd-task.md) -- [RsyncTask](remote-cmd-task.md) +- [RsyncTask](rsync-task.md) - [Checkers (HttpChecker, PortChecker, and PathChecker)](checkers.md): Check parent task's readiness. As every task are extended from `BaseTask`, you will see that most of them share some common parameters. diff --git a/docs/concepts/tasks/remote-cmd-task.md b/docs/concepts/tasks/remote-cmd-task.md index 7ec8093b..5b1dcacb 100644 --- a/docs/concepts/tasks/remote-cmd-task.md +++ b/docs/concepts/tasks/remote-cmd-task.md @@ -27,4 +27,13 @@ install_curl = RemoteCmdTask( runner.register(install_curl) ``` +RemoteCmdTask exposes several environments that you can use on your `cmd` and `cmd_path` + +- `_CONFIG_HOST` +- `_CONFIG_PORT` +- `_CONFIG_SSH_KEY` +- `_CONFIG_USER` +- `_CONFIG_PASSWORD` +- `_CONFIG_MAP_` + 🔖 [Table of Contents](../../README.md) / [Concepts](../README.md) / [Tasks](README.md) From ad10f959dc9c4096cb2578f5ff231faef8cdd2cb Mon Sep 17 00:00:00 2001 From: goFrendiAsgard Date: Tue, 7 Nov 2023 13:03:33 +0700 Subject: [PATCH 11/18] expose env --- src/zrb/task/base_remote_cmd_task.py | 58 ++++++++++++++++++++-------- 1 file changed, 42 insertions(+), 16 deletions(-) diff --git a/src/zrb/task/base_remote_cmd_task.py b/src/zrb/task/base_remote_cmd_task.py index eba8487d..722a1833 100644 --- a/src/zrb/task/base_remote_cmd_task.py +++ b/src/zrb/task/base_remote_cmd_task.py @@ -1,5 +1,5 @@ from zrb.helper.typing import ( - Any, Callable, Iterable, Mapping, Optional, Union, TypeVar + Any, Callable, Iterable, List, Mapping, Optional, Union, TypeVar ) from zrb.helper.typecheck import typechecked from zrb.helper.util import to_snake_case @@ -122,28 +122,54 @@ def __init__( self._post_cmd = post_cmd self._post_cmd_path = post_cmd_path self._remote_config = remote_config + self._is_additional_env_added = False def copy(self) -> TSingleBaseRemoteCmdTask: return copy.deepcopy(self) - def _get_shell_env_map(self) -> Mapping[str, Any]: - env_map = super()._get_shell_env_map() - env_map['_CONFIG_HOST'] = self.render_str(self._remote_config.host) - env_map['_CONFIG_PORT'] = str(self.render_int( - self._remote_config.port) - ) - env_map['_CONFIG_SSH_KEY'] = self.render_str( - self._remote_config.ssh_key - ) - env_map['_CONFIG_USER'] = self.render_str(self._remote_config.user) - env_map['_CONFIG_PASSWORD'] = self.render_str( - self._remote_config.password - ) + def _get_all_envs(self) -> Mapping[str, Env]: + ''' + This method override CmdTask's _get_all_envs. + Whenever _get_all_envs is called, we want to make sure that + we also include generated environment based on remote_config + ''' + if self._is_additional_env_added: + return super()._get_all_envs() + self._is_additional_env_added = True + additional_envs: List[Env] = [ + Env( + name='_CONFIG_HOST', os_name='', + default=self.render_str(self._remote_config.host) + ), + Env( + name='_CONFIG_PORT', os_name='', + default=str(self.render_int(self._remote_config.port)) + ), + Env( + name='_CONFIG_SSH_KEY', os_name='', + default=self.render_str(self._remote_config.ssh_key) + ), + Env( + name='_CONFIG_USER', os_name='', + default=self.render_str(self._remote_config.user) + ), + Env( + name='_CONFIG_PASSWORD', os_name='', + default=self.render_str(self._remote_config.password) + ), + ] for key, val in self._remote_config.config_map.items(): upper_snake_key = to_snake_case(key).upper() rendered_val = self.render_str(val) - env_map['_CONFIG_MAP_' + upper_snake_key] = rendered_val - return env_map + additional_envs.append( + Env( + name='_CONFIG_MAP_' + upper_snake_key, + os_name='', + default=rendered_val + ) + ) + self._envs = list(self._envs) + additional_envs + return super()._get_all_envs() def _get_cmd_str(self, *args: Any, **kwargs: Any) -> str: cmd_str = '\n'.join([ From 6611e2c90bd9a9f9704329da00ab39d8f9bab37b Mon Sep 17 00:00:00 2001 From: goFrendiAsgard Date: Wed, 8 Nov 2023 17:56:42 +0700 Subject: [PATCH 12/18] use _get_all_envs --- docs/concepts/tasks/README.md | 8 ++++---- docs/tutorials/copy-and-reuse-task.md | 4 ++-- src/zrb/task/any_task.py | 8 ++++---- src/zrb/task/base_task.py | 11 ++++++----- src/zrb/task/base_task_composite.py | 18 +++++++++--------- src/zrb/task/cmd_task.py | 20 ++++++++++++-------- src/zrb/task/docker_compose_task.py | 6 +++--- src/zrb/task/flow_task.py | 8 ++++---- zrb_init.py | 6 +++--- 9 files changed, 47 insertions(+), 42 deletions(-) diff --git a/docs/concepts/tasks/README.md b/docs/concepts/tasks/README.md index 87a21cd3..7829f6f6 100644 --- a/docs/concepts/tasks/README.md +++ b/docs/concepts/tasks/README.md @@ -400,13 +400,13 @@ Every task share some common methods like `run`, `check`, and `to_function`. Deep copy current task -## `add_envs` +## `add_env` -## `add_env_files` +## `add_env_file` -## `add_inputs` +## `add_input` -## `add_upstreams` +## `add_upstream` ## `set_name` diff --git a/docs/tutorials/copy-and-reuse-task.md b/docs/tutorials/copy-and-reuse-task.md index f50c1032..8c7a64f3 100644 --- a/docs/tutorials/copy-and-reuse-task.md +++ b/docs/tutorials/copy-and-reuse-task.md @@ -20,10 +20,10 @@ local_hello = hello.copy() # Update name, input, and env local_hello.set_name('hello-local') -local_hello.add_inputs( +local_hello.add_input( StrInput(name='name', description='Name', default='dunia') ) -local_hello.add_envs( +local_hello.add_env( Env(name='GREETINGS', os_name='', default='Halo') ) diff --git a/src/zrb/task/any_task.py b/src/zrb/task/any_task.py index 78bb19a6..36869dd6 100644 --- a/src/zrb/task/any_task.py +++ b/src/zrb/task/any_task.py @@ -65,19 +65,19 @@ def to_function( pass @abstractmethod - def add_upstreams(self, *upstreams: TAnyTask): + def add_upstream(self, *upstreams: TAnyTask): pass @abstractmethod - def add_inputs(self, *inputs: AnyInput): + def add_input(self, *inputs: AnyInput): pass @abstractmethod - def add_envs(self, *envs: Env): + def add_env(self, *envs: Env): pass @abstractmethod - def add_env_files(self, *env_files: EnvFile): + def add_env_file(self, *env_files: EnvFile): pass @abstractmethod diff --git a/src/zrb/task/base_task.py b/src/zrb/task/base_task.py index 169d9c39..2871a911 100644 --- a/src/zrb/task/base_task.py +++ b/src/zrb/task/base_task.py @@ -126,14 +126,15 @@ def get_all_inputs(self) -> Iterable[AnyInput]: self._all_inputs: List[AnyInput] = [] existing_input_names: Mapping[str, bool] = {} # Add task inputs - for input_index, first_occurence_task_input in enumerate(self._inputs): + inputs = self.get_inputs() + for input_index, first_occurence_task_input in enumerate(inputs): input_name = first_occurence_task_input.get_name() if input_name in existing_input_names: continue # Look for all input with the same name in the current task task_inputs = [ candidate - for candidate in self._inputs[input_index:] + for candidate in inputs[input_index:] if candidate.get_name() == input_name ] # Get the last input, and add it to _all_inputs @@ -167,7 +168,7 @@ def function(*args: Any, **kwargs: Any) -> Any: )) return function - def add_upstreams(self, *upstreams: AnyTask): + def add_upstream(self, *upstreams: AnyTask): if not self._allow_add_upstreams: raise Exception(f'Cannot add upstreams on `{self._name}`') self._upstreams += upstreams @@ -288,10 +289,10 @@ def _get_all_envs(self) -> Mapping[str, Env]: all_envs[env_name] = Env( name=env_name, os_name=env_name, renderable=False ) - for env_file in self._env_files: + for env_file in self.get_env_files(): for env in env_file.get_envs(): all_envs[env.name] = env - for env in self._envs: + for env in self.get_envs(): all_envs[env.name] = env return all_envs diff --git a/src/zrb/task/base_task_composite.py b/src/zrb/task/base_task_composite.py index a5510dd2..3211f097 100644 --- a/src/zrb/task/base_task_composite.py +++ b/src/zrb/task/base_task_composite.py @@ -99,17 +99,17 @@ def set_retry_interval(self, new_retry_interval: Union[float, int]): def set_checking_interval(self, new_checking_interval: Union[float, int]): self._checking_interval = new_checking_interval - def add_inputs(self, *inputs: AnyInput): + def add_input(self, *inputs: AnyInput): if not self._allow_add_inputs: raise Exception(f'Cannot add inputs on `{self._name}`') self._inputs += inputs - def add_envs(self, *envs: Env): + def add_env(self, *envs: Env): if not self._allow_add_envs: raise Exception(f'Cannot add envs on `{self._name}`') self._envs += envs - def add_env_files(self, *env_files: EnvFile): + def add_env_file(self, *env_files: EnvFile): if not self._allow_add_env_files: raise Exception(f'Cannot add env_files on `{self._name}`') self._env_files += env_files @@ -127,16 +127,16 @@ def get_env_files(self) -> List[EnvFile]: return self._env_files def get_envs(self) -> List[Env]: - return self._envs + return list(self._envs) def get_inputs(self) -> List[AnyInput]: - return self._inputs + return list(self._inputs) - def get_checkers(self) -> Iterable[AnyTask]: - return self._checkers + def get_checkers(self) -> List[AnyTask]: + return list(self._checkers) - def get_upstreams(self) -> Iterable[AnyTask]: - return self._upstreams + def get_upstreams(self) -> List[AnyTask]: + return list(self._upstreams) def get_description(self) -> str: return self._description diff --git a/src/zrb/task/cmd_task.py b/src/zrb/task/cmd_task.py index 756e6dd2..dbd4a62e 100644 --- a/src/zrb/task/cmd_task.py +++ b/src/zrb/task/cmd_task.py @@ -160,6 +160,7 @@ def __init__( self._executable = executable self._process: Optional[asyncio.subprocess.Process] self._preexec_fn = preexec_fn + self._is_cmd_aditional_env_added = False def copy(self) -> TCmdTask: return super().copy() @@ -182,18 +183,21 @@ def print_result(self, result: CmdResult): return print(result.output) - def _get_shell_env_map(self) -> Mapping[str, Any]: - env_map = self.get_env_map() + def _get_all_envs(self) -> Mapping[str, Env]: + if self._is_cmd_aditional_env_added: + return super()._get_all_envs() input_map = self.get_input_map() + additional_envs: List[Env] = [] for input_name, input_value in input_map.items(): - upper_input_name = '_INPUT_' + input_name.upper() - if upper_input_name not in env_map: - env_map[upper_input_name] = f'{input_value}' - return env_map + env_name = '_INPUT_' + input_name.upper() + additional_envs.append( + Env(name=env_name, os_name='', default=str(input_value)) + ) + self._envs += additional_envs + return super()._get_all_envs() async def run(self, *args: Any, **kwargs: Any) -> CmdResult: cmd = self._get_cmd_str(*args, **kwargs) - env_map = self._get_shell_env_map() self.print_out_dark('Run script: ' + self._get_multiline_repr(cmd)) self.print_out_dark('Working directory: ' + self._cwd) self._output_buffer = [] @@ -203,7 +207,7 @@ async def run(self, *args: Any, **kwargs: Any) -> CmdResult: cwd=self._cwd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, - env=env_map, + env=self.get_env_map(), shell=True, executable=self._executable, close_fds=True, diff --git a/src/zrb/task/docker_compose_task.py b/src/zrb/task/docker_compose_task.py index 117683df..99a47195 100644 --- a/src/zrb/task/docker_compose_task.py +++ b/src/zrb/task/docker_compose_task.py @@ -153,7 +153,7 @@ def __init__( ) # Flag to make mark whether service config and compose environments # has been added to this task's envs and env_files - self._is_additional_env_added = False + self._is_compose_additional_env_added = False def copy(self) -> TDockerComposeTask: return super().copy() @@ -173,9 +173,9 @@ def _get_all_envs(self) -> Mapping[str, Env]: - Service config's envs and env_files are included - Any environment defined in docker compose file is also included ''' - if self._is_additional_env_added: + if self._is_compose_additional_env_added: return super()._get_all_envs() - self._is_additional_env_added = True + self._is_compose_additional_env_added = True # define additional envs and additonal env_files additional_envs: List[Env] = [] additional_env_files: List[EnvFile] = [] diff --git a/src/zrb/task/flow_task.py b/src/zrb/task/flow_task.py index 5dd138c5..601f2e2b 100644 --- a/src/zrb/task/flow_task.py +++ b/src/zrb/task/flow_task.py @@ -107,10 +107,10 @@ def _get_embeded_tasks( embeded_tasks: List[AnyTask] = [] for task in tasks: embeded_task = task.copy() - embeded_task.add_upstreams(*upstreams) - embeded_task.add_envs(*envs) - embeded_task.add_env_files(*env_files) - embeded_task.add_inputs(*inputs) + embeded_task.add_upstream(*upstreams) + embeded_task.add_env(*envs) + embeded_task.add_env_file(*env_files) + embeded_task.add_input(*inputs) embeded_tasks.append(embeded_task) return embeded_tasks diff --git a/zrb_init.py b/zrb_init.py index 0c7d5b28..7b04e760 100644 --- a/zrb_init.py +++ b/zrb_init.py @@ -99,7 +99,7 @@ ], ) skippable_build: CmdTask = build.copy() -skippable_build.add_inputs(build_zrb_input) +skippable_build.add_input(build_zrb_input) skippable_build.set_should_execute('{{ input.build_zrb}}') runner.register(build) @@ -273,7 +273,7 @@ ] ) skippable_install_symlink: CmdTask = install_symlink.copy() -skippable_install_symlink.add_inputs( +skippable_install_symlink.add_input( build_zrb_input, install_symlink_input ) @@ -342,7 +342,7 @@ preexec_fn=None ) skippable_create_playground: CmdTask = create_playground.copy() -skippable_create_playground.add_inputs( +skippable_create_playground.add_input( build_zrb_input, install_symlink_input, create_playground_input From 7b36178d25b556c1e31c167512a6a175fb69ff89 Mon Sep 17 00:00:00 2001 From: goFrendiAsgard Date: Wed, 8 Nov 2023 18:04:08 +0700 Subject: [PATCH 13/18] remove direct property call --- src/zrb/task/base_task.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/zrb/task/base_task.py b/src/zrb/task/base_task.py index 2871a911..730fca45 100644 --- a/src/zrb/task/base_task.py +++ b/src/zrb/task/base_task.py @@ -142,7 +142,7 @@ def get_all_inputs(self) -> Iterable[AnyInput]: self._all_inputs.append(task_input) existing_input_names[input_name] = True # Add upstream inputs - for upstream in self._upstreams: + for upstream in self.get_upstreams(): upstream_inputs = upstream.get_all_inputs() for upstream_input in upstream_inputs: if upstream_input.get_name() in existing_input_names: @@ -351,9 +351,10 @@ def _print_result(self, result: Any): return if self._return_upstream_result: # if _return_upstream_result, result is list (see: self._run_all) + upstreams = self.get_upstreams() upstream_results = list(result) for upstream_index, upstream_result in enumerate(upstream_results): - self._upstreams[upstream_index]._print_result(upstream_result) + upstreams[upstream_index]._print_result(upstream_result) return self.print_result(result) @@ -452,7 +453,7 @@ async def _run_all(self, *args: Any, **kwargs: Any) -> Any: coroutines: Iterable[asyncio.Task] = [] # Add upstream tasks to processes self._allow_add_upstreams = False - for upstream_task in self._upstreams: + for upstream_task in self.get_upstreams(): upstream_task.set_execution_id(self.get_execution_id()) coroutines.append(asyncio.create_task( upstream_task._run_all(**kwargs) @@ -475,7 +476,7 @@ async def _cached_run(self, *args: Any, **kwargs: Any) -> Any: # get upstream checker upstream_check_processes: Iterable[asyncio.Task] = [] self._allow_add_upstreams = False - for upstream_task in self._upstreams: + for upstream_task in self.get_upstreams(): upstream_check_processes.append(asyncio.create_task( upstream_task._loop_check() )) @@ -530,9 +531,9 @@ async def _set_keyval(self, kwargs: Mapping[str, Any], env_prefix: str): new_kwargs = copy.deepcopy(kwargs) new_kwargs.update(self.get_input_map()) upstream_coroutines = [] - # set uplstreams keyval + # set upstreams keyval self._allow_add_upstreams = False - for upstream_task in self._upstreams: + for upstream_task in self.get_upstreams(): upstream_coroutines.append(asyncio.create_task( upstream_task._set_keyval( kwargs=new_kwargs, env_prefix=env_prefix @@ -542,7 +543,7 @@ async def _set_keyval(self, kwargs: Mapping[str, Any], env_prefix: str): local_env_map = self.get_env_map() checker_coroutines = [] for checker_task in self._checkers: - checker_task._inputs += self._inputs + checker_task.add_input(*self.get_inputs()) checker_task.inject_env_map(local_env_map, override=True) checker_coroutines.append(asyncio.create_task( checker_task._set_keyval( From 294567c0592665920ea32e6c436a8bb010fbca68 Mon Sep 17 00:00:00 2001 From: goFrendiAsgard Date: Wed, 8 Nov 2023 19:29:51 +0700 Subject: [PATCH 14/18] add insert_task --- src/zrb/task/any_task.py | 12 ++++++++++++ src/zrb/task/base_task.py | 10 ++++++++-- src/zrb/task/base_task_composite.py | 15 +++++++++++++++ 3 files changed, 35 insertions(+), 2 deletions(-) diff --git a/src/zrb/task/any_task.py b/src/zrb/task/any_task.py index 36869dd6..9bfb6053 100644 --- a/src/zrb/task/any_task.py +++ b/src/zrb/task/any_task.py @@ -68,14 +68,26 @@ def to_function( def add_upstream(self, *upstreams: TAnyTask): pass + @abstractmethod + def insert_input(self, *inputs: AnyInput): + pass + @abstractmethod def add_input(self, *inputs: AnyInput): pass + @abstractmethod + def insert_env(self, *envs: Env): + pass + @abstractmethod def add_env(self, *envs: Env): pass + @abstractmethod + def insert_env_file(self, *env_files: EnvFile): + pass + @abstractmethod def add_env_file(self, *env_files: EnvFile): pass diff --git a/src/zrb/task/base_task.py b/src/zrb/task/base_task.py index 730fca45..ab93fbbb 100644 --- a/src/zrb/task/base_task.py +++ b/src/zrb/task/base_task.py @@ -168,6 +168,11 @@ def function(*args: Any, **kwargs: Any) -> Any: )) return function + def insert_upstream(self, *upstreams: AnyTask): + if not self._allow_add_upstreams: + raise Exception(f'Cannot add upstreams on `{self._name}`') + self._upstreams = upstreams + self._upstreams + def add_upstream(self, *upstreams: AnyTask): if not self._allow_add_upstreams: raise Exception(f'Cannot add upstreams on `{self._name}`') @@ -540,11 +545,12 @@ async def _set_keyval(self, kwargs: Mapping[str, Any], env_prefix: str): ) )) # set checker keyval - local_env_map = self.get_env_map() + # local_env_map = self.get_env_map() checker_coroutines = [] for checker_task in self._checkers: checker_task.add_input(*self.get_inputs()) - checker_task.inject_env_map(local_env_map, override=True) + checker_task.add_env(*self.get_envs()) + # checker_task.inject_env_map(local_env_map, override=True) checker_coroutines.append(asyncio.create_task( checker_task._set_keyval( kwargs=new_kwargs, env_prefix=env_prefix diff --git a/src/zrb/task/base_task_composite.py b/src/zrb/task/base_task_composite.py index 3211f097..874eb9cd 100644 --- a/src/zrb/task/base_task_composite.py +++ b/src/zrb/task/base_task_composite.py @@ -99,16 +99,31 @@ def set_retry_interval(self, new_retry_interval: Union[float, int]): def set_checking_interval(self, new_checking_interval: Union[float, int]): self._checking_interval = new_checking_interval + def insert_input(self, *inputs: AnyInput): + if not self._allow_add_inputs: + raise Exception(f'Cannot add inputs on `{self._name}`') + self._inputs = inputs + self._inputs + def add_input(self, *inputs: AnyInput): if not self._allow_add_inputs: raise Exception(f'Cannot add inputs on `{self._name}`') self._inputs += inputs + def insert_env(self, *envs: Env): + if not self._allow_add_envs: + raise Exception(f'Cannot add envs on `{self._name}`') + self._envs = envs + self._envs + def add_env(self, *envs: Env): if not self._allow_add_envs: raise Exception(f'Cannot add envs on `{self._name}`') self._envs += envs + def insert_env_file(self, *env_files: EnvFile): + if not self._allow_add_env_files: + raise Exception(f'Cannot add env_files on `{self._name}`') + self._env_files = env_files + self._env_files + def add_env_file(self, *env_files: EnvFile): if not self._allow_add_env_files: raise Exception(f'Cannot add env_files on `{self._name}`') From 0d5724179a54ba8010a0e34268b4183d9850c1cf Mon Sep 17 00:00:00 2001 From: goFrendiAsgard Date: Thu, 9 Nov 2023 08:39:00 +0700 Subject: [PATCH 15/18] rename methods --- docs/concepts/tasks/README.md | 8 +++++ docs/quirks.md | 1 - docs/tutorials/extending-cmd-task.md | 30 ++++++++++------ src/zrb/action/runner.py | 2 +- src/zrb/task/any_task.py | 4 +-- src/zrb/task/base_remote_cmd_task.py | 28 +++++++-------- src/zrb/task/base_task.py | 35 +++++++------------ src/zrb/task/base_task_composite.py | 35 ++++++++++++------- src/zrb/task/cmd_task.py | 37 ++++++++++---------- src/zrb/task/docker_compose_task.py | 51 ++++++++++++---------------- 10 files changed, 120 insertions(+), 111 deletions(-) diff --git a/docs/concepts/tasks/README.md b/docs/concepts/tasks/README.md index 7829f6f6..2d48ae54 100644 --- a/docs/concepts/tasks/README.md +++ b/docs/concepts/tasks/README.md @@ -400,12 +400,20 @@ Every task share some common methods like `run`, `check`, and `to_function`. Deep copy current task +## `insert_env` + ## `add_env` +## `insert_env_file` + ## `add_env_file` +## `insert_input` + ## `add_input` +## `insert_upstream` + ## `add_upstream` ## `set_name` diff --git a/docs/quirks.md b/docs/quirks.md index c0a8d075..c8d50375 100644 --- a/docs/quirks.md +++ b/docs/quirks.md @@ -18,6 +18,5 @@ - `env` will override each other, the last one takes greater priority - If you define a `DockerComposeTask`, it will automatically fill your environment with the ones you use in your docker-compose file. The environment defined that way will have a very low priority. They will be overridden by both `env_files` and `env`. - You cannot have an input named: `_task`, `_args` or `_execution_id` -- You cannot have an environment named `_execution_id` 🔖 [Table of Contents](README.md) diff --git a/docs/tutorials/extending-cmd-task.md b/docs/tutorials/extending-cmd-task.md index d73333dd..0559f421 100644 --- a/docs/tutorials/extending-cmd-task.md +++ b/docs/tutorials/extending-cmd-task.md @@ -55,19 +55,29 @@ class SlackPrintTask(CmdTask): self._slack_channel_id = slack_channel_id self._slack_app_token = slack_app_token self._message = message + self._slack_env_added = False - def run(self, *args: Any, **kwargs: Any): - # Inject environment variables - self.inject_env_map( - env_map={ - 'CHANNEL_ID': self.render_str(self._slack_channel_id), - 'TOKEN': self.render_str(self._slack_app_token), - 'MESSAGE': self.render_str(self._message) - } + def get_envs(self) -> List[Env]: + if self._slack_env_added: + return super().get_env() + self._slack_env_added = True + self.add_envs( + Env( + name='CHANNEL_ID', os_name='', + default=self.render_str(self._slack_channel_id) + ), + Env( + name='TOKEN', os_name='', + default=self.render_str(self._slack_app_token) + ), + Env( + name='MESSAGE', os_name='', + default=self.render_str(self._message) + ) ) - return super().run(*args, **kwargs) + return super().get_env() - def _get_cmd_str(self, *args: Any, **kwargs: Any): + def get_cmd_script(self, *args: Any, **kwargs: Any): # contruct json payload and replace all `"` with `\\"` json_payload = jsons.dumps({ 'channel': '$CHANNEL_ID', diff --git a/src/zrb/action/runner.py b/src/zrb/action/runner.py index 4767ed4c..11f26585 100644 --- a/src/zrb/action/runner.py +++ b/src/zrb/action/runner.py @@ -77,7 +77,7 @@ def _get_cli_group(self, task_group: TaskGroup) -> click.Group: return group def _create_cli_command(self, task: AnyTask) -> click.Command: - task_inputs = task.get_all_inputs() + task_inputs = task._get_combined_inputs() task_cmd_name = task.get_cmd_name() task_description = task.get_description() task_function = task.to_function( diff --git a/src/zrb/task/any_task.py b/src/zrb/task/any_task.py index 9bfb6053..2cf19e6e 100644 --- a/src/zrb/task/any_task.py +++ b/src/zrb/task/any_task.py @@ -175,7 +175,7 @@ def get_upstreams(self) -> Iterable[TAnyTask]: pass @abstractmethod - def get_all_inputs(self) -> Iterable[AnyInput]: + def _get_combined_inputs(self) -> Iterable[AnyInput]: pass @abstractmethod @@ -219,7 +219,7 @@ def get_env_map(self) -> Mapping[str, Any]: pass @abstractmethod - def inject_env_map( + def _inject_env_map( self, env_map: Mapping[str, str], override: bool = False ): pass diff --git a/src/zrb/task/base_remote_cmd_task.py b/src/zrb/task/base_remote_cmd_task.py index 722a1833..68e33333 100644 --- a/src/zrb/task/base_remote_cmd_task.py +++ b/src/zrb/task/base_remote_cmd_task.py @@ -127,16 +127,12 @@ def __init__( def copy(self) -> TSingleBaseRemoteCmdTask: return copy.deepcopy(self) - def _get_all_envs(self) -> Mapping[str, Env]: - ''' - This method override CmdTask's _get_all_envs. - Whenever _get_all_envs is called, we want to make sure that - we also include generated environment based on remote_config - ''' + def get_envs(self) -> List[Env]: if self._is_additional_env_added: - return super()._get_all_envs() + return super().get_envs() self._is_additional_env_added = True - additional_envs: List[Env] = [ + # add remote config properties as env + self.add_env( Env( name='_CONFIG_HOST', os_name='', default=self.render_str(self._remote_config.host) @@ -157,27 +153,27 @@ def _get_all_envs(self) -> Mapping[str, Env]: name='_CONFIG_PASSWORD', os_name='', default=self.render_str(self._remote_config.password) ), - ] + ) for key, val in self._remote_config.config_map.items(): upper_snake_key = to_snake_case(key).upper() rendered_val = self.render_str(val) - additional_envs.append( + # add remote config map as env + self.add_env( Env( name='_CONFIG_MAP_' + upper_snake_key, os_name='', default=rendered_val ) ) - self._envs = list(self._envs) + additional_envs - return super()._get_all_envs() + return super().get_envs() - def _get_cmd_str(self, *args: Any, **kwargs: Any) -> str: + def get_cmd_script(self, *args: Any, **kwargs: Any) -> str: cmd_str = '\n'.join([ - self._create_cmd_str( + self._create_cmd_script( self._pre_cmd_path, self._pre_cmd, *args, **kwargs ), - super()._get_cmd_str(*args, **kwargs), - self._create_cmd_str( + super().get_cmd_script(*args, **kwargs), + self._create_cmd_script( self._post_cmd_path, self._post_cmd, *args, **kwargs ), ]) diff --git a/src/zrb/task/base_task.py b/src/zrb/task/base_task.py index ab93fbbb..4220165f 100644 --- a/src/zrb/task/base_task.py +++ b/src/zrb/task/base_task.py @@ -110,19 +110,16 @@ def __init__( self._is_execution_started: bool = False self._args: List[Any] = [] self._kwargs: Mapping[str, Any] = {} - self._allow_add_upstreams: bool = True def copy(self) -> AnyTask: return copy.deepcopy(self) - def get_all_inputs(self) -> Iterable[AnyInput]: + def _get_combined_inputs(self) -> Iterable[AnyInput]: '''' Getting all inputs of this task and all its upstream, non-duplicated. ''' if self._all_inputs is not None: return self._all_inputs - self._allow_add_upstreams = False - self._allow_add_inputs = False self._all_inputs: List[AnyInput] = [] existing_input_names: Mapping[str, bool] = {} # Add task inputs @@ -143,12 +140,14 @@ def get_all_inputs(self) -> Iterable[AnyInput]: existing_input_names[input_name] = True # Add upstream inputs for upstream in self.get_upstreams(): - upstream_inputs = upstream.get_all_inputs() + upstream_inputs = upstream._get_combined_inputs() for upstream_input in upstream_inputs: if upstream_input.get_name() in existing_input_names: continue self._all_inputs.append(upstream_input) existing_input_names[upstream_input.get_name()] = True + self._allow_add_upstreams = False + self._allow_add_inputs = False return self._all_inputs def to_function( @@ -168,17 +167,7 @@ def function(*args: Any, **kwargs: Any) -> Any: )) return function - def insert_upstream(self, *upstreams: AnyTask): - if not self._allow_add_upstreams: - raise Exception(f'Cannot add upstreams on `{self._name}`') - self._upstreams = upstreams + self._upstreams - - def add_upstream(self, *upstreams: AnyTask): - if not self._allow_add_upstreams: - raise Exception(f'Cannot add upstreams on `{self._name}`') - self._upstreams += upstreams - - def inject_env_map( + def _inject_env_map( self, env_map: Mapping[str, str], override: bool = False ): ''' @@ -265,7 +254,7 @@ async def _set_local_keyval( return True self._is_keyval_set = True self.log_info('Set input map') - for task_input in self.get_all_inputs(): + for task_input in self._get_combined_inputs(): input_name = self._get_normalized_input_key(task_input.get_name()) input_value = self.render_any( kwargs.get(input_name, task_input.get_default()) @@ -275,7 +264,7 @@ async def _set_local_keyval( 'Input map:\n' + map_to_str(self.get_input_map(), item_prefix=' ') ) self.log_info('Merging task envs, task env files, and native envs') - for env_name, env in self._get_all_envs().items(): + for env_name, env in self._get_combined_env().items(): env_value = env.get(env_prefix) if env.renderable: env_value = self.render_any(env_value) @@ -284,9 +273,7 @@ async def _set_local_keyval( 'Env map:\n' + map_to_str(self.get_env_map(), item_prefix=' ') ) - def _get_all_envs(self) -> Mapping[str, Env]: - self._allow_add_envs = False - self._allow_add_env_files = False + def _get_combined_env(self) -> Mapping[str, Env]: all_envs: Mapping[str, Env] = {} for env_name in os.environ: if env_name in RESERVED_ENV_NAMES: @@ -299,6 +286,8 @@ def _get_all_envs(self) -> Mapping[str, Env]: all_envs[env.name] = env for env in self.get_envs(): all_envs[env.name] = env + self._allow_add_envs = False + self._allow_add_env_files = False return all_envs def _get_normalized_input_key(self, key: str) -> str: @@ -402,7 +391,7 @@ def _show_env_prefix(self): def _show_run_command(self): params: List[str] = [double_quote(arg) for arg in self._args] - for task_input in self.get_all_inputs(): + for task_input in self._get_combined_inputs(): if task_input.is_hidden(): continue key = task_input.get_name() @@ -525,7 +514,7 @@ async def _check_should_execute(self, *args: Any, **kwargs: Any) -> bool: async def _set_keyval(self, kwargs: Mapping[str, Any], env_prefix: str): # if input is not in input_map, add default values - for task_input in self.get_all_inputs(): + for task_input in self._get_combined_inputs(): key = self._get_normalized_input_key(task_input.get_name()) if key in kwargs: continue diff --git a/src/zrb/task/base_task_composite.py b/src/zrb/task/base_task_composite.py index 874eb9cd..4c986b5b 100644 --- a/src/zrb/task/base_task_composite.py +++ b/src/zrb/task/base_task_composite.py @@ -66,6 +66,7 @@ def __init__( self._allow_add_envs = True self._allow_add_env_files = True self._allow_add_inputs = True + self._allow_add_upstreams: bool = True self._execution_id = '' def set_execution_id(self, execution_id: str): @@ -101,33 +102,43 @@ def set_checking_interval(self, new_checking_interval: Union[float, int]): def insert_input(self, *inputs: AnyInput): if not self._allow_add_inputs: - raise Exception(f'Cannot add inputs on `{self._name}`') - self._inputs = inputs + self._inputs + raise Exception(f'Cannot insert inputs for `{self._name}`') + self._inputs = list(inputs) + list(self._inputs) def add_input(self, *inputs: AnyInput): if not self._allow_add_inputs: - raise Exception(f'Cannot add inputs on `{self._name}`') - self._inputs += inputs + raise Exception(f'Cannot add inputs for `{self._name}`') + self._inputs = list(self._inputs) + list(inputs) def insert_env(self, *envs: Env): if not self._allow_add_envs: - raise Exception(f'Cannot add envs on `{self._name}`') - self._envs = envs + self._envs + raise Exception(f'Cannot insert envs to `{self._name}`') + self._envs = list(envs) + list(self._envs) def add_env(self, *envs: Env): if not self._allow_add_envs: - raise Exception(f'Cannot add envs on `{self._name}`') - self._envs += envs + raise Exception(f'Cannot add envs to `{self._name}`') + self._envs = list(self._envs) + list(envs) def insert_env_file(self, *env_files: EnvFile): if not self._allow_add_env_files: - raise Exception(f'Cannot add env_files on `{self._name}`') - self._env_files = env_files + self._env_files + raise Exception(f'Cannot insert env_files to `{self._name}`') + self._env_files = list(env_files) + list(self._env_files) def add_env_file(self, *env_files: EnvFile): if not self._allow_add_env_files: - raise Exception(f'Cannot add env_files on `{self._name}`') - self._env_files += env_files + raise Exception(f'Cannot add env_files to `{self._name}`') + self._env_files = list(self._env_files) + list(env_files) + + def insert_upstream(self, *upstreams: AnyTask): + if not self._allow_add_upstreams: + raise Exception(f'Cannot insert upstreams to `{self._name}`') + self._upstreams = list(upstreams) + list(self._upstreams) + + def add_upstream(self, *upstreams: AnyTask): + if not self._allow_add_upstreams: + raise Exception(f'Cannot add upstreams to `{self._name}`') + self._upstreams = list(self._upstreams) + list(upstreams) def get_execution_id(self) -> str: return self._execution_id diff --git a/src/zrb/task/cmd_task.py b/src/zrb/task/cmd_task.py index dbd4a62e..2a9ba957 100644 --- a/src/zrb/task/cmd_task.py +++ b/src/zrb/task/cmd_task.py @@ -1,5 +1,5 @@ from zrb.helper.typing import ( - Any, Callable, Iterable, List, Mapping, Optional, Union, TypeVar + Any, Callable, Iterable, List, Optional, Union, TypeVar ) from zrb.helper.typecheck import typechecked from zrb.task.any_task import AnyTask @@ -183,21 +183,20 @@ def print_result(self, result: CmdResult): return print(result.output) - def _get_all_envs(self) -> Mapping[str, Env]: + def get_envs(self) -> List[Env]: if self._is_cmd_aditional_env_added: - return super()._get_all_envs() + return super().get_envs() + self._is_cmd_aditional_env_added = True input_map = self.get_input_map() - additional_envs: List[Env] = [] for input_name, input_value in input_map.items(): env_name = '_INPUT_' + input_name.upper() - additional_envs.append( + self.add_env( Env(name=env_name, os_name='', default=str(input_value)) ) - self._envs += additional_envs - return super()._get_all_envs() + return super().get_envs() async def run(self, *args: Any, **kwargs: Any) -> CmdResult: - cmd = self._get_cmd_str(*args, **kwargs) + cmd = self.get_cmd_script(*args, **kwargs) self.print_out_dark('Run script: ' + self._get_multiline_repr(cmd)) self.print_out_dark('Working directory: ' + self._cwd) self._output_buffer = [] @@ -318,21 +317,25 @@ async def _wait_process(self, process: asyncio.subprocess.Process): await stdout_log_process await stderr_log_process - def _get_cmd_str(self, *args: Any, **kwargs: Any) -> str: - return self._create_cmd_str(self._cmd_path, self._cmd, *args, **kwargs) + def get_cmd_script(self, *args: Any, **kwargs: Any) -> str: + return self._create_cmd_script( + self._cmd_path, self._cmd, *args, **kwargs + ) - def _create_cmd_str( + def _create_cmd_script( self, cmd_path: CmdVal, cmd: CmdVal, *args: Any, **kwargs: Any ) -> str: if not isinstance(cmd_path, str) or cmd_path != '': if callable(cmd_path): - return self._render_cmd_path_str(cmd_path(*args, **kwargs)) - return self._render_cmd_path_str(cmd_path) + return self._get_rendered_cmd_path(cmd_path(*args, **kwargs)) + return self._get_rendered_cmd_path(cmd_path) if callable(cmd): - return self._render_cmd_str(cmd(*args, **kwargs)) - return self._render_cmd_str(cmd) + return self._get_rendered_cmd(cmd(*args, **kwargs)) + return self._get_rendered_cmd(cmd) - def _render_cmd_path_str(self, cmd_path: Union[str, Iterable[str]]) -> str: + def _get_rendered_cmd_path( + self, cmd_path: Union[str, Iterable[str]] + ) -> str: if isinstance(cmd_path, str): return self.render_file(cmd_path) return '\n'.join([ @@ -340,7 +343,7 @@ def _render_cmd_path_str(self, cmd_path: Union[str, Iterable[str]]) -> str: for cmd_path_str in cmd_path ]) - def _render_cmd_str(self, cmd: Union[str, Iterable[str]]) -> str: + def _get_rendered_cmd(self, cmd: Union[str, Iterable[str]]) -> str: if isinstance(cmd, str): return self.render_str(cmd) return self.render_str('\n'.join(list(cmd))) diff --git a/src/zrb/task/docker_compose_task.py b/src/zrb/task/docker_compose_task.py index 99a47195..a3c4c76c 100644 --- a/src/zrb/task/docker_compose_task.py +++ b/src/zrb/task/docker_compose_task.py @@ -154,6 +154,7 @@ def __init__( # Flag to make mark whether service config and compose environments # has been added to this task's envs and env_files self._is_compose_additional_env_added = False + self._is_compose_additional_env_file_added = False def copy(self) -> TDockerComposeTask: return super().copy() @@ -166,45 +167,37 @@ async def run(self, *args, **kwargs: Any) -> CmdResult: os.remove(self._compose_runtime_file) return result - def _get_all_envs(self) -> Mapping[str, Env]: - ''' - This method override BaseTask's _get_all_envs. - Whenever _get_all_envs is called, we want to make sure that: - - Service config's envs and env_files are included - - Any environment defined in docker compose file is also included - ''' + def get_envs(self) -> List[Env]: if self._is_compose_additional_env_added: - return super()._get_all_envs() + return super().get_envs() self._is_compose_additional_env_added = True - # define additional envs and additonal env_files - additional_envs: List[Env] = [] - additional_env_files: List[EnvFile] = [] - # populate additional envs and additional env_files - # with service configs + # inject envs from service_configs for _, service_config in self._compose_service_configs.items(): - additional_env_files += service_config.get_env_files() - additional_envs += service_config.get_envs() - # populate additional envs and additional env_files with - # compose envs - data = read_compose_file(self._compose_template_file) - env_map = fetch_compose_file_env_map(data) + self.insert_env(*service_config.get_envs()) + # inject envs from docker compose file + compose_data = read_compose_file(self._compose_template_file) + env_map = fetch_compose_file_env_map(compose_data) added_env_map: Mapping[str, bool] = {} for key, value in env_map.items(): # Need to get this everytime because we only want # the first compose file env value for a certain key if key in RESERVED_ENV_NAMES or key in added_env_map: continue + added_env_map[key] = True os_name = key if self._compose_env_prefix != '': os_name = f'{self._compose_env_prefix}_{os_name}' - compose_env = Env(name=key, os_name=os_name, default=value) - additional_envs.append(compose_env) - added_env_map[key] = True - # Add additional envs and addition env files to this task - self._envs = additional_envs + list(self._envs) - self._env_files = additional_env_files + list(self._env_files) - # get all envs - return super()._get_all_envs() + self.insert_env(Env(name=key, os_name=os_name, default=value)) + return super().get_envs() + + def get_env_files(self) -> List[EnvFile]: + if self._is_compose_additional_env_file_added: + return super().get_env_file() + self._is_compose_additional_env_file_added = True + # inject env_files from service_configs + for _, service_config in self._compose_service_configs.items(): + self.insert_env_file(*service_config.get_env_files()) + return super().get_env_files() def _generate_compose_runtime_file(self): compose_data = read_compose_file(self._compose_template_file) @@ -311,8 +304,8 @@ def _get_compose_template_file(self, compose_file: Optional[str]) -> str: return os.path.join(self._cwd, compose_file) raise Exception(f'Invalid compose file: {compose_file}') - def _get_cmd_str(self, *args: Any, **kwargs: Any) -> str: - setup_cmd_str = self._create_cmd_str( + def get_cmd_script(self, *args: Any, **kwargs: Any) -> str: + setup_cmd_str = self._create_cmd_script( self._setup_cmd_path, self._setup_cmd, *args, **kwargs ) command_options = dict(self._compose_options) From c151f0c3833b0e8c738d90645656bb106b327b7a Mon Sep 17 00:00:00 2001 From: goFrendiAsgard Date: Thu, 9 Nov 2023 10:52:22 +0700 Subject: [PATCH 16/18] make more methods protected --- src/zrb/action/runner.py | 2 +- src/zrb/helper/env_map/fetch.py | 26 ++++++++-------- src/zrb/task/any_task.py | 20 +++++------- src/zrb/task/base_remote_cmd_task.py | 6 ++-- src/zrb/task/base_task.py | 46 ++++++++++------------------ src/zrb/task/base_task_composite.py | 35 +++++++++++---------- src/zrb/task/cmd_task.py | 6 ++-- src/zrb/task/docker_compose_task.py | 10 +++--- 8 files changed, 66 insertions(+), 85 deletions(-) diff --git a/src/zrb/action/runner.py b/src/zrb/action/runner.py index 11f26585..10663178 100644 --- a/src/zrb/action/runner.py +++ b/src/zrb/action/runner.py @@ -28,7 +28,7 @@ def __init__(self, env_prefix: str = ''): def register(self, task: AnyTask): task.set_has_cli_interface() - cmd_name = task.get_complete_cmd_name() + cmd_name = task.get_full_cmd_name() logger.debug(colored(f'Register task: {cmd_name}', attrs=['dark'])) self._tasks.append(task) logger.debug(colored(f'Task registered: {cmd_name}', attrs=['dark'])) diff --git a/src/zrb/helper/env_map/fetch.py b/src/zrb/helper/env_map/fetch.py index 3633d2bf..a3fd1c21 100644 --- a/src/zrb/helper/env_map/fetch.py +++ b/src/zrb/helper/env_map/fetch.py @@ -16,7 +16,7 @@ def fetch_env_map_from_group( sub_env_map: Mapping[str, str] = fetch_env_map_from_group( env_map, sub_group ) - env_map = cascade_env_map(env_map, sub_env_map) + env_map = _cascade_env_map(env_map, sub_env_map) return env_map @@ -25,33 +25,33 @@ def fetch_env_map_from_task( env_map: Mapping[str, str], task: AnyTask ): task_env_map: Mapping[str, str] = {} - for env_file in task.get_env_files(): + for env_file in task._get_env_files(): envs = env_file.get_envs() - task_env_map = add_envs_to_env_map(task_env_map, envs) - task_env_map = add_envs_to_env_map(task_env_map, task._envs) - env_map = cascade_env_map(env_map, task_env_map) - for upstream in task.get_upstreams(): + task_env_map = _add_envs_to_env_map(task_env_map, envs) + task_env_map = _add_envs_to_env_map(task_env_map, task._envs) + env_map = _cascade_env_map(env_map, task_env_map) + for upstream in task._get_upstreams(): task_env_map = fetch_env_map_from_task(env_map, upstream) - for checker in task.get_checkers(): + for checker in task._get_checkers(): task_env_map = fetch_env_map_from_task(env_map, checker) return env_map @typechecked -def add_envs_to_env_map( +def _add_envs_to_env_map( env_map: Mapping[str, str], envs: List[Env] ) -> Mapping[str, str]: for env in envs: if env.os_name == '': continue - env_name = get_env_name(env) - env_default = get_env_default(env) + env_name = _get_env_name(env) + env_default = _get_env_default(env) env_map[env_name] = env_default return env_map @typechecked -def cascade_env_map( +def _cascade_env_map( env_map: Mapping[str, str], other_env_map: Mapping[str, str] ) -> Mapping[str, str]: @@ -63,14 +63,14 @@ def cascade_env_map( @typechecked -def get_env_name(env: Env) -> str: +def _get_env_name(env: Env) -> str: if env.os_name is None: return env.name return env.os_name @typechecked -def get_env_default(env: Env) -> str: +def _get_env_default(env: Env) -> str: if is_probably_jinja(env.default): return '' return env.default diff --git a/src/zrb/task/any_task.py b/src/zrb/task/any_task.py index 2cf19e6e..29ecf4d3 100644 --- a/src/zrb/task/any_task.py +++ b/src/zrb/task/any_task.py @@ -93,7 +93,7 @@ def add_env_file(self, *env_files: EnvFile): pass @abstractmethod - def set_execution_id(self, execution_id: str): + def _set_execution_id(self, execution_id: str): pass @abstractmethod @@ -151,27 +151,27 @@ def get_cmd_name(self) -> str: pass @abstractmethod - def get_complete_cmd_name(self) -> str: + def get_full_cmd_name(self) -> str: pass @abstractmethod - def get_env_files(self) -> List[EnvFile]: + def _get_env_files(self) -> List[EnvFile]: pass @abstractmethod - def get_envs(self) -> List[Env]: + def _get_envs(self) -> List[Env]: pass @abstractmethod - def get_inputs(self) -> List[AnyInput]: + def _get_inputs(self) -> List[AnyInput]: pass @abstractmethod - def get_checkers(self) -> Iterable[TAnyTask]: + def _get_checkers(self) -> Iterable[TAnyTask]: pass @abstractmethod - def get_upstreams(self) -> Iterable[TAnyTask]: + def _get_upstreams(self) -> Iterable[TAnyTask]: pass @abstractmethod @@ -218,12 +218,6 @@ def get_input_map(self) -> Mapping[str, Any]: def get_env_map(self) -> Mapping[str, Any]: pass - @abstractmethod - def _inject_env_map( - self, env_map: Mapping[str, str], override: bool = False - ): - pass - @abstractmethod def render_any( self, val: Any, data: Optional[Mapping[str, Any]] = None diff --git a/src/zrb/task/base_remote_cmd_task.py b/src/zrb/task/base_remote_cmd_task.py index 68e33333..c79caae0 100644 --- a/src/zrb/task/base_remote_cmd_task.py +++ b/src/zrb/task/base_remote_cmd_task.py @@ -127,9 +127,9 @@ def __init__( def copy(self) -> TSingleBaseRemoteCmdTask: return copy.deepcopy(self) - def get_envs(self) -> List[Env]: + def _get_envs(self) -> List[Env]: if self._is_additional_env_added: - return super().get_envs() + return super()._get_envs() self._is_additional_env_added = True # add remote config properties as env self.add_env( @@ -165,7 +165,7 @@ def get_envs(self) -> List[Env]: default=rendered_val ) ) - return super().get_envs() + return super()._get_envs() def get_cmd_script(self, *args: Any, **kwargs: Any) -> str: cmd_str = '\n'.join([ diff --git a/src/zrb/task/base_task.py b/src/zrb/task/base_task.py index 4220165f..8730bfa6 100644 --- a/src/zrb/task/base_task.py +++ b/src/zrb/task/base_task.py @@ -123,7 +123,7 @@ def _get_combined_inputs(self) -> Iterable[AnyInput]: self._all_inputs: List[AnyInput] = [] existing_input_names: Mapping[str, bool] = {} # Add task inputs - inputs = self.get_inputs() + inputs = self._get_inputs() for input_index, first_occurence_task_input in enumerate(inputs): input_name = first_occurence_task_input.get_name() if input_name in existing_input_names: @@ -139,7 +139,7 @@ def _get_combined_inputs(self) -> Iterable[AnyInput]: self._all_inputs.append(task_input) existing_input_names[input_name] = True # Add upstream inputs - for upstream in self.get_upstreams(): + for upstream in self._get_upstreams(): upstream_inputs = upstream._get_combined_inputs() for upstream_input in upstream_inputs: if upstream_input.get_name() in existing_input_names: @@ -167,16 +167,6 @@ def function(*args: Any, **kwargs: Any) -> Any: )) return function - def _inject_env_map( - self, env_map: Mapping[str, str], override: bool = False - ): - ''' - Set new values for current task's env map - ''' - for key, val in env_map.items(): - if override or key not in self.get_env_map(): - self._set_env_map(key, val) - async def run(self, *args: Any, **kwargs: Any) -> Any: ''' Do task execution @@ -260,6 +250,7 @@ async def _set_local_keyval( kwargs.get(input_name, task_input.get_default()) ) self._set_input_map(input_name, input_value) + self._set_input_map('_execution_id', self._execution_id) self.log_debug( 'Input map:\n' + map_to_str(self.get_input_map(), item_prefix=' ') ) @@ -269,6 +260,7 @@ async def _set_local_keyval( if env.renderable: env_value = self.render_any(env_value) self._set_env_map(env_name, env_value) + self._set_env_map('_ZRB_EXECUTION_ID', self._execution_id) self.log_debug( 'Env map:\n' + map_to_str(self.get_env_map(), item_prefix=' ') ) @@ -281,10 +273,10 @@ def _get_combined_env(self) -> Mapping[str, Env]: all_envs[env_name] = Env( name=env_name, os_name=env_name, renderable=False ) - for env_file in self.get_env_files(): + for env_file in self._get_env_files(): for env in env_file.get_envs(): all_envs[env.name] = env - for env in self.get_envs(): + for env in self._get_envs(): all_envs[env.name] = env self._allow_add_envs = False self._allow_add_env_files = False @@ -305,7 +297,7 @@ async def _run_and_check_all( try: self._start_timer() if self.get_execution_id() == '': - self.set_execution_id( + self._set_execution_id( get_random_name(add_random_digit=True, digit_count=5) ) self.log_info('Set input and env map') @@ -345,7 +337,7 @@ def _print_result(self, result: Any): return if self._return_upstream_result: # if _return_upstream_result, result is list (see: self._run_all) - upstreams = self.get_upstreams() + upstreams = self._get_upstreams() upstream_results = list(result) for upstream_index, upstream_result in enumerate(upstream_results): upstreams[upstream_index]._print_result(upstream_result) @@ -361,11 +353,6 @@ def print_result(self, result: Any): if you want to show the result differently. ''' print(result) - - def set_execution_id(self, execution_id: str): - super().set_execution_id(execution_id) - self._set_env_map('_ZRB_EXECUTION_ID', execution_id) - self._set_input_map('_execution_id', execution_id) async def _loop_check(self, show_done: bool = False) -> bool: self.log_info('Start readiness checking') @@ -398,7 +385,7 @@ def _show_run_command(self): kwarg_key = self._get_normalized_input_key(key) quoted_value = double_quote(str(self._kwargs[kwarg_key])) params.append(f'--{key} {quoted_value}') - run_cmd = self.get_complete_cmd_name() + run_cmd = self.get_full_cmd_name() run_cmd_with_param = run_cmd if len(params) > 0: param_str = ' '.join(params) @@ -435,7 +422,7 @@ async def _check(self) -> bool: await asyncio.sleep(0.1) check_coroutines: Iterable[asyncio.Task] = [] for checker_task in self._checkers: - checker_task.set_execution_id(self.get_execution_id()) + checker_task._set_execution_id(self.get_execution_id()) check_coroutines.append( asyncio.create_task(checker_task._run_all()) ) @@ -447,8 +434,8 @@ async def _run_all(self, *args: Any, **kwargs: Any) -> Any: coroutines: Iterable[asyncio.Task] = [] # Add upstream tasks to processes self._allow_add_upstreams = False - for upstream_task in self.get_upstreams(): - upstream_task.set_execution_id(self.get_execution_id()) + for upstream_task in self._get_upstreams(): + upstream_task._set_execution_id(self.get_execution_id()) coroutines.append(asyncio.create_task( upstream_task._run_all(**kwargs) )) @@ -470,7 +457,7 @@ async def _cached_run(self, *args: Any, **kwargs: Any) -> Any: # get upstream checker upstream_check_processes: Iterable[asyncio.Task] = [] self._allow_add_upstreams = False - for upstream_task in self.get_upstreams(): + for upstream_task in self._get_upstreams(): upstream_check_processes.append(asyncio.create_task( upstream_task._loop_check() )) @@ -527,7 +514,7 @@ async def _set_keyval(self, kwargs: Mapping[str, Any], env_prefix: str): upstream_coroutines = [] # set upstreams keyval self._allow_add_upstreams = False - for upstream_task in self.get_upstreams(): + for upstream_task in self._get_upstreams(): upstream_coroutines.append(asyncio.create_task( upstream_task._set_keyval( kwargs=new_kwargs, env_prefix=env_prefix @@ -537,9 +524,8 @@ async def _set_keyval(self, kwargs: Mapping[str, Any], env_prefix: str): # local_env_map = self.get_env_map() checker_coroutines = [] for checker_task in self._checkers: - checker_task.add_input(*self.get_inputs()) - checker_task.add_env(*self.get_envs()) - # checker_task.inject_env_map(local_env_map, override=True) + checker_task.add_input(*self._get_inputs()) + checker_task.add_env(*self._get_envs()) checker_coroutines.append(asyncio.create_task( checker_task._set_keyval( kwargs=new_kwargs, env_prefix=env_prefix diff --git a/src/zrb/task/base_task_composite.py b/src/zrb/task/base_task_composite.py index 4c986b5b..05cccd6d 100644 --- a/src/zrb/task/base_task_composite.py +++ b/src/zrb/task/base_task_composite.py @@ -69,8 +69,9 @@ def __init__( self._allow_add_upstreams: bool = True self._execution_id = '' - def set_execution_id(self, execution_id: str): - self._execution_id = execution_id + def _set_execution_id(self, execution_id: str): + if self._execution_id != '': + self._execution_id = execution_id def set_name(self, new_name: str): if self._description == self._name: @@ -149,19 +150,19 @@ def get_icon(self) -> str: def get_color(self) -> str: return self._color - def get_env_files(self) -> List[EnvFile]: + def _get_env_files(self) -> List[EnvFile]: return self._env_files - def get_envs(self) -> List[Env]: + def _get_envs(self) -> List[Env]: return list(self._envs) - def get_inputs(self) -> List[AnyInput]: + def _get_inputs(self) -> List[AnyInput]: return list(self._inputs) - def get_checkers(self) -> List[AnyTask]: + def _get_checkers(self) -> List[AnyTask]: return list(self._checkers) - def get_upstreams(self) -> List[AnyTask]: + def _get_upstreams(self) -> List[AnyTask]: return list(self._upstreams) def get_description(self) -> str: @@ -387,7 +388,7 @@ def __init__( run: Optional[Callable[..., Any]] = None, should_execute: Union[bool, str, Callable[..., bool]] = True ): - self._filled_complete_name: Optional[str] = None + self._rjust_full_cmd_name: Optional[str] = None self._has_cli_interface = False self._complete_name: Optional[str] = None CommonTaskModel.__init__( @@ -474,13 +475,13 @@ def _get_colored(self, text: str) -> str: def _get_print_prefix(self) -> str: common_prefix = self._get_common_prefix(show_time=show_time) icon = self.get_icon() - truncated_name = self._get_filled_complete_name() + truncated_name = self._get_rjust_full_cmd_name() return f'{common_prefix} {icon} {truncated_name}' def _get_log_prefix(self) -> str: common_prefix = self._get_common_prefix(show_time=False) icon = self.get_icon() - filled_name = self._get_filled_complete_name() + filled_name = self._get_rjust_full_cmd_name() return f'{common_prefix} {icon} {filled_name}' def _get_common_prefix(self, show_time: bool) -> str: @@ -492,14 +493,14 @@ def _get_common_prefix(self, show_time: bool) -> str: return f'◷ {now} ❁ {pid} → {attempt}/{max_attempt}' return f'❁ {pid} → {attempt}/{max_attempt}' - def _get_filled_complete_name(self) -> str: - if self._filled_complete_name is not None: - return self._filled_complete_name - complete_name = self.get_complete_cmd_name() - self._filled_complete_name = complete_name.rjust(LOG_NAME_LENGTH, ' ') - return self._filled_complete_name + def _get_rjust_full_cmd_name(self) -> str: + if self._rjust_full_cmd_name is not None: + return self._rjust_full_cmd_name + complete_name = self.get_full_cmd_name() + self._rjust_full_cmd_name = complete_name.rjust(LOG_NAME_LENGTH, ' ') + return self._rjust_full_cmd_name - def get_complete_cmd_name(self) -> str: + def get_full_cmd_name(self) -> str: if self._complete_name is not None: return self._complete_name executable_prefix = '' diff --git a/src/zrb/task/cmd_task.py b/src/zrb/task/cmd_task.py index 2a9ba957..b9a42aaa 100644 --- a/src/zrb/task/cmd_task.py +++ b/src/zrb/task/cmd_task.py @@ -183,9 +183,9 @@ def print_result(self, result: CmdResult): return print(result.output) - def get_envs(self) -> List[Env]: + def _get_envs(self) -> List[Env]: if self._is_cmd_aditional_env_added: - return super().get_envs() + return super()._get_envs() self._is_cmd_aditional_env_added = True input_map = self.get_input_map() for input_name, input_value in input_map.items(): @@ -193,7 +193,7 @@ def get_envs(self) -> List[Env]: self.add_env( Env(name=env_name, os_name='', default=str(input_value)) ) - return super().get_envs() + return super()._get_envs() async def run(self, *args: Any, **kwargs: Any) -> CmdResult: cmd = self.get_cmd_script(*args, **kwargs) diff --git a/src/zrb/task/docker_compose_task.py b/src/zrb/task/docker_compose_task.py index a3c4c76c..9c28d791 100644 --- a/src/zrb/task/docker_compose_task.py +++ b/src/zrb/task/docker_compose_task.py @@ -167,9 +167,9 @@ async def run(self, *args, **kwargs: Any) -> CmdResult: os.remove(self._compose_runtime_file) return result - def get_envs(self) -> List[Env]: + def _get_envs(self) -> List[Env]: if self._is_compose_additional_env_added: - return super().get_envs() + return super()._get_envs() self._is_compose_additional_env_added = True # inject envs from service_configs for _, service_config in self._compose_service_configs.items(): @@ -188,16 +188,16 @@ def get_envs(self) -> List[Env]: if self._compose_env_prefix != '': os_name = f'{self._compose_env_prefix}_{os_name}' self.insert_env(Env(name=key, os_name=os_name, default=value)) - return super().get_envs() + return super()._get_envs() - def get_env_files(self) -> List[EnvFile]: + def _get_env_files(self) -> List[EnvFile]: if self._is_compose_additional_env_file_added: return super().get_env_file() self._is_compose_additional_env_file_added = True # inject env_files from service_configs for _, service_config in self._compose_service_configs.items(): self.insert_env_file(*service_config.get_env_files()) - return super().get_env_files() + return super()._get_env_files() def _generate_compose_runtime_file(self): compose_data = read_compose_file(self._compose_template_file) From 5bc4e417f7830f0f0ba17b5d39dc1e38ac29bbd0 Mon Sep 17 00:00:00 2001 From: goFrendiAsgard Date: Thu, 9 Nov 2023 12:48:59 +0700 Subject: [PATCH 17/18] introduce inject methods --- docs/tutorials/extending-cmd-task.md | 7 +----- src/zrb/task/any_task.py | 20 ++++++++++++++++ src/zrb/task/base_remote_cmd_task.py | 10 +++----- src/zrb/task/base_task_composite.py | 35 ++++++++++++++++++++++++++++ src/zrb/task/cmd_task.py | 8 ++----- src/zrb/task/docker_compose_task.py | 13 +++-------- 6 files changed, 64 insertions(+), 29 deletions(-) diff --git a/docs/tutorials/extending-cmd-task.md b/docs/tutorials/extending-cmd-task.md index 0559f421..1c78bbd0 100644 --- a/docs/tutorials/extending-cmd-task.md +++ b/docs/tutorials/extending-cmd-task.md @@ -55,12 +55,8 @@ class SlackPrintTask(CmdTask): self._slack_channel_id = slack_channel_id self._slack_app_token = slack_app_token self._message = message - self._slack_env_added = False - def get_envs(self) -> List[Env]: - if self._slack_env_added: - return super().get_env() - self._slack_env_added = True + def inject_envs(self): self.add_envs( Env( name='CHANNEL_ID', os_name='', @@ -75,7 +71,6 @@ class SlackPrintTask(CmdTask): default=self.render_str(self._message) ) ) - return super().get_env() def get_cmd_script(self, *args: Any, **kwargs: Any): # contruct json payload and replace all `"` with `\\"` diff --git a/src/zrb/task/any_task.py b/src/zrb/task/any_task.py index 29ecf4d3..14c482ae 100644 --- a/src/zrb/task/any_task.py +++ b/src/zrb/task/any_task.py @@ -154,22 +154,42 @@ def get_cmd_name(self) -> str: def get_full_cmd_name(self) -> str: pass + @abstractmethod + def inject_env_files(self): + pass + @abstractmethod def _get_env_files(self) -> List[EnvFile]: pass + @abstractmethod + def inject_envs(self): + pass + @abstractmethod def _get_envs(self) -> List[Env]: pass + @abstractmethod + def inject_inputs(self): + pass + @abstractmethod def _get_inputs(self) -> List[AnyInput]: pass + @abstractmethod + def inject_checkers(self): + pass + @abstractmethod def _get_checkers(self) -> Iterable[TAnyTask]: pass + @abstractmethod + def inject_upstreams(self): + pass + @abstractmethod def _get_upstreams(self) -> Iterable[TAnyTask]: pass diff --git a/src/zrb/task/base_remote_cmd_task.py b/src/zrb/task/base_remote_cmd_task.py index c79caae0..af5c0977 100644 --- a/src/zrb/task/base_remote_cmd_task.py +++ b/src/zrb/task/base_remote_cmd_task.py @@ -1,5 +1,5 @@ from zrb.helper.typing import ( - Any, Callable, Iterable, List, Mapping, Optional, Union, TypeVar + Any, Callable, Iterable, Mapping, Optional, Union, TypeVar ) from zrb.helper.typecheck import typechecked from zrb.helper.util import to_snake_case @@ -122,15 +122,12 @@ def __init__( self._post_cmd = post_cmd self._post_cmd_path = post_cmd_path self._remote_config = remote_config - self._is_additional_env_added = False def copy(self) -> TSingleBaseRemoteCmdTask: return copy.deepcopy(self) - def _get_envs(self) -> List[Env]: - if self._is_additional_env_added: - return super()._get_envs() - self._is_additional_env_added = True + def inject_envs(self): + super().inject_envs() # add remote config properties as env self.add_env( Env( @@ -165,7 +162,6 @@ def _get_envs(self) -> List[Env]: default=rendered_val ) ) - return super()._get_envs() def get_cmd_script(self, *args: Any, **kwargs: Any) -> str: cmd_str = '\n'.join([ diff --git a/src/zrb/task/base_task_composite.py b/src/zrb/task/base_task_composite.py index 05cccd6d..80a069cf 100644 --- a/src/zrb/task/base_task_composite.py +++ b/src/zrb/task/base_task_composite.py @@ -67,6 +67,11 @@ def __init__( self._allow_add_env_files = True self._allow_add_inputs = True self._allow_add_upstreams: bool = True + self._has_already_inject_env_files: bool = False + self._has_already_inject_envs: bool = False + self._has_already_inject_inputs: bool = False + self._has_already_inject_checkers: bool = False + self._has_already_inject_upstreams: bool = False self._execution_id = '' def _set_execution_id(self, execution_id: str): @@ -150,19 +155,49 @@ def get_icon(self) -> str: def get_color(self) -> str: return self._color + def inject_env_files(self): + pass + def _get_env_files(self) -> List[EnvFile]: + if not self._has_already_inject_env_files: + self.inject_env_files() + self._has_already_inject_env_files = True return self._env_files + def inject_envs(self): + pass + def _get_envs(self) -> List[Env]: + if not self._has_already_inject_envs: + self.inject_envs() + self._has_already_inject_envs = True return list(self._envs) + def inject_inputs(self): + pass + def _get_inputs(self) -> List[AnyInput]: + if not self._has_already_inject_inputs: + self.inject_inputs() + self._has_already_inject_inputs = True return list(self._inputs) + def inject_checkers(self): + pass + def _get_checkers(self) -> List[AnyTask]: + if not self._has_already_inject_checkers: + self.inject_checkers() + self._has_already_inject_checkers = True return list(self._checkers) + def inject_upstreams(self): + pass + def _get_upstreams(self) -> List[AnyTask]: + if not self._has_already_inject_upstreams: + self.inject_upstreams() + self._has_already_inject_upstreams = True return list(self._upstreams) def get_description(self) -> str: diff --git a/src/zrb/task/cmd_task.py b/src/zrb/task/cmd_task.py index b9a42aaa..417988af 100644 --- a/src/zrb/task/cmd_task.py +++ b/src/zrb/task/cmd_task.py @@ -160,7 +160,6 @@ def __init__( self._executable = executable self._process: Optional[asyncio.subprocess.Process] self._preexec_fn = preexec_fn - self._is_cmd_aditional_env_added = False def copy(self) -> TCmdTask: return super().copy() @@ -183,17 +182,14 @@ def print_result(self, result: CmdResult): return print(result.output) - def _get_envs(self) -> List[Env]: - if self._is_cmd_aditional_env_added: - return super()._get_envs() - self._is_cmd_aditional_env_added = True + def inject_envs(self): + super().inject_envs() input_map = self.get_input_map() for input_name, input_value in input_map.items(): env_name = '_INPUT_' + input_name.upper() self.add_env( Env(name=env_name, os_name='', default=str(input_value)) ) - return super()._get_envs() async def run(self, *args: Any, **kwargs: Any) -> CmdResult: cmd = self.get_cmd_script(*args, **kwargs) diff --git a/src/zrb/task/docker_compose_task.py b/src/zrb/task/docker_compose_task.py index 9c28d791..f70d3060 100644 --- a/src/zrb/task/docker_compose_task.py +++ b/src/zrb/task/docker_compose_task.py @@ -167,10 +167,8 @@ async def run(self, *args, **kwargs: Any) -> CmdResult: os.remove(self._compose_runtime_file) return result - def _get_envs(self) -> List[Env]: - if self._is_compose_additional_env_added: - return super()._get_envs() - self._is_compose_additional_env_added = True + def inject_envs(self): + super().inject_envs() # inject envs from service_configs for _, service_config in self._compose_service_configs.items(): self.insert_env(*service_config.get_envs()) @@ -188,16 +186,11 @@ def _get_envs(self) -> List[Env]: if self._compose_env_prefix != '': os_name = f'{self._compose_env_prefix}_{os_name}' self.insert_env(Env(name=key, os_name=os_name, default=value)) - return super()._get_envs() - def _get_env_files(self) -> List[EnvFile]: - if self._is_compose_additional_env_file_added: - return super().get_env_file() - self._is_compose_additional_env_file_added = True + def inject_env_files(self): # inject env_files from service_configs for _, service_config in self._compose_service_configs.items(): self.insert_env_file(*service_config.get_env_files()) - return super()._get_env_files() def _generate_compose_runtime_file(self): compose_data = read_compose_file(self._compose_template_file) From 9c3790557e66a862d985eec15596c3908af352ca Mon Sep 17 00:00:00 2001 From: goFrendiAsgard Date: Thu, 9 Nov 2023 12:52:48 +0700 Subject: [PATCH 18/18] adjust docs --- docs/concepts/tasks/README.md | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/docs/concepts/tasks/README.md b/docs/concepts/tasks/README.md index 2d48ae54..8ca7580c 100644 --- a/docs/concepts/tasks/README.md +++ b/docs/concepts/tasks/README.md @@ -400,22 +400,46 @@ Every task share some common methods like `run`, `check`, and `to_function`. Deep copy current task +## `inject_env` + +To be overridden + ## `insert_env` ## `add_env` +## `inject_env_file` + +To be overridden + ## `insert_env_file` ## `add_env_file` +## `inject_input` + +To be overridden + ## `insert_input` ## `add_input` +## `inject_upstream` + +To be overridden + ## `insert_upstream` ## `add_upstream` +## `inject_checker` + +To be overridden + +## `insert_checker` + +## `add_checker` + ## `set_name` ## `set_description`