diff --git a/.github/workflows/_zrb.yml b/.github/workflows/_zrb.yml index a1493bd8..babce1f8 100644 --- a/.github/workflows/_zrb.yml +++ b/.github/workflows/_zrb.yml @@ -17,7 +17,7 @@ jobs: Run-command: runs-on: ubuntu-latest container: - image: stalchmst/zrb:0.0.106 + image: stalchmst/zrb:0.0.107 steps: - name: Check out repository code uses: actions/checkout@v3 diff --git a/README.md b/README.md index 9d979bdb..8b53cafc 100644 --- a/README.md +++ b/README.md @@ -2,12 +2,14 @@ ![](https://raw.githubusercontent.com/state-alchemists/zrb/main/images/zrb/android-chrome-192x192.png) -[📖 Documentation](https://github.com/state-alchemists/zrb/blob/main/docs/README.md) | [🏁 Getting started](https://github.com/state-alchemists/zrb/blob/main/docs/getting-started.md) +[📖 Documentation](https://github.com/state-alchemists/zrb/blob/main/docs/README.md) | [🏁 Getting Started](https://github.com/state-alchemists/zrb/blob/main/docs/getting-started.md) | [💃 Oops, I did it Again](https://github.com/state-alchemists/zrb/blob/main/docs/oops-i-did-it-again/README.md)| [❓ FAQ](https://github.com/state-alchemists/zrb/blob/main/docs/faq/README.md) -Zrb is a [CLI-based](https://en.wikipedia.org/wiki/Command-line_interface) automation [tool](https://en.wikipedia.org/wiki/Programming_tool) and [low-code](https://en.wikipedia.org/wiki/Low-code_development_platform) platform. Once installed, you can automate day-to-day tasks, generate projects and applications, and even deploy your applications to Kubernetes with a few commands. +Zrb is a [CLI-based](https://en.wikipedia.org/wiki/Command-line_interface) automation [tool](https://en.wikipedia.org/wiki/Programming_tool) and [low-code](https://en.wikipedia.org/wiki/Low-code_development_platform) platform. Once installed, Zrb will help you automate day-to-day tasks, generate projects and applications, and even deploy your applications to Kubernetes with a few commands. To use Zrb, you need to be familiar with CLI. +Zrb task definitions are written in [Python](https://www.python.org/), and we have a [very good reason](https://github.com/state-alchemists/zrb/blob/main/docs/faq/why-python.md) behind the decision. + ## Zrb as a low-code framework Let's see how you can build and run a [CRUD](https://en.wikipedia.org/wiki/Create,_read,_update_and_delete) application. diff --git a/docs/README.md b/docs/README.md index fe67a0f0..1b6fc738 100644 --- a/docs/README.md +++ b/docs/README.md @@ -5,6 +5,7 @@ - [Configurations](configurations.md) - [Tutorials](tutorials/README.md) - [Troubleshooting](troubleshooting/README.md) -- [For contributors](for-contributors.md) -- [For maintainers](for-maintainers.md) -- [FAQ](faq.md) +- [Oops, I Did it Again: Most Common Mistakes When Working with Zrb](oops-i-did-it-again/README.md) +- [Contributor Guide](contributor-guide.md) +- [Maintainer Guide](maintainer-guide.md) +- [FAQ](faq/README.md) diff --git a/docs/for-contributors.md b/docs/contributor-guide.md similarity index 96% rename from docs/for-contributors.md rename to docs/contributor-guide.md index 6a58dbf2..31d5bd2f 100644 --- a/docs/for-contributors.md +++ b/docs/contributor-guide.md @@ -1,6 +1,6 @@ 🔖 [Table of Contents](README.md) -# For contributors +# Contributor Guide As contributors, there are some things you can do: diff --git a/docs/faq.md b/docs/faq.md deleted file mode 100644 index a1f736f9..00000000 --- a/docs/faq.md +++ /dev/null @@ -1,14 +0,0 @@ -🔖 [Table of Contents](README.md) - -# FAQ - -## How to spell Zrb? - -You can spell Zrb as `Zaruba`. - -## Why Python? - -Python is a multipurpose language. It gives you a lot of flexibility to define your tasks without being too verbose. - - -🔖 [Table of Contents](README.md) \ No newline at end of file diff --git a/docs/faq/README.md b/docs/faq/README.md new file mode 100644 index 00000000..4bf26dce --- /dev/null +++ b/docs/faq/README.md @@ -0,0 +1,7 @@ +🔖 [Table of Contents](../README.md) + +# FAQ + +- [Why Python](why-python.md) + +🔖 [Table of Contents](../README.md) diff --git a/docs/faq/why-python.md b/docs/faq/why-python.md new file mode 100644 index 00000000..abc37135 --- /dev/null +++ b/docs/faq/why-python.md @@ -0,0 +1,115 @@ +🔖 [Table of Contents](../README.md) / [FAQ](README.md) + +# Why Python? + +Python is a general multi-purpose language. It support a lot of pogramming paradigms like OOP/FP, or procedural. Writing a configuration in Python let you do a lot of things like control structure, etc. + +Python has been around since the 90's, and you won't have to worry about the language. Many people already familiar with the language. Even if you are new to the language, learning Python will be highly rewarding. + +However, there are a lot of tools out there that don't use Python. So, why Python? Why not YAML/HCL/anything else? + +# Why not YAML? + +YAML format is widely used because of it's readability and simplicity. Many tools like [Ansible](https://docs.ansible.com/ansible/latest/playbook_guide/playbooks_intro.html) or [Docker Compose](https://docs.docker.com/compose/) use YAML files for their configuration. + +Using YAML for simple configuration can be a good idea. + +However, for more complex scenario (like when you need to do looping or branching), using YAML is quite challenging. You need to overcome this with some tricks: + +- Generating the YAML configuration somewhere else +- Using templating language like Jinja or Go template +- Implementing branch/loop structure in your YAML parser configuration + +Compared to this, Python already has built-in control structure. + +Moreover, you can define your Python configuration declaratively, almost like YAML. + +Let's see how Ansible playbook and Zrb config might like similar to each other: + +## Ansible Playbook + +First of all, you need to define your virtual machines + +```ini +# file: vm_inventory.ini +[my_vms] +192.168.1.100 ansible_ssh_user=ubuntu +192.168.1.101 ansible_ssh_user=ubuntu +192.168.1.102 ansible_ssh_user=ubuntu +``` + +Then you create a playbook to define what you want to do with the virtual machines + +```yaml +# file: install_curl.yml +--- +- name: Install curl on VMs + hosts: my_vms + become: true + tasks: + - name: Update package cache + apt: + update_cache: yes + + - name: Install curl + apt: + name: curl + state: present +``` + +Finally, you run the playbook agains the virtual machines: + +```bash +ansible-playbook -i vm_inventory.ini install_curl.yml +``` + +## Zrb Task Definition + +You can define something similar with Zrb: + +```python +# file: zrb_init.py +from zrb import ( + runner, CmdTask, RemoteCmdTask, RemoteConfig, PasswordInput +) + +remote_configs = [ + RemoteConfig( + host=f'192.168.1.{sub_ip}', + user='ubuntu' + ) for sub_ip in range(100, 103) +] + +update_package_cache = RemoteCmdTask( + name='update-package-cache', + remote_configs=remote_configs, + cmd='sudo apt update' +) + +install_curl = RemoteCmdTask( + name='install-curl', + remote_configs=remote_configs, + upstreams=[update_package_cache], + cmd='sudo apt install curl --y' +) +runner.register(install_curl) +``` + +Then you can run the the task by invoking: + +```bash +zrb install-curl +``` + +## Comparing Ansible Playbook and Zrb Task Definition + +If you are not familiar with Python, Ansible Playbook will makes more sense for you. Ansible task definition is carefully crafted to cover most use cases. + +The trickiest part when you work with Ansible or any YAML based configuration is you need to understand the specification. Docker compose for example, has a very different configuration from ansible eventough both of them are using YAML. + +On the other hand, Python is just Python. You can use list comprehension, loop, branch, or anything you already know. The syntax highlighting, hint, and auto completion are commonly provided in your favorite tools. + +Even if you are new to Python, Zrb Task Definition is not very difficult to grasp. You can follow the [getting started guide](../getting-started.md) and tag along. + + +🔖 [Table of Contents](../README.md) / [FAQ](README.md) diff --git a/docs/getting-started.md b/docs/getting-started.md index ca483a20..e1370424 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -6,11 +6,15 @@ Zrb is an automation tool. With Zrb you can run tasks using command-line-interfa There are project tasks and common tasks. Project tasks are usually bind to a project, while common tasks can be executed from anywhere. -# Running a common task +# Running a task -To run a common task, you can type `zrb [task-groups] [task-parameters]`. +You can run any Zrb task by invoking the following pattern: -For example, you want to run `encode` task under `base64` group, you can do so by execute the following: +```bash +zrb [task-groups] [task-parameters] +``` + +For example, you want to run `encode` that is located under `base64` group, you can do so by execute the following command: ```bash zrb base64 encode --text "non-credential-string" @@ -32,7 +36,9 @@ Related tasks are usually located under the same group. For example, you have `d zrb base64 decode --text "bm9uLWNyZWRlbnRpYWwtc3RyaW5n" ``` -Don't worry if you can't remember all available `task-group`, `task-name`, or `task-parameters`. Just press enter at any time, and Zrb will show you the way. +# Getting available tasks/task groups + +To see all available task/task groups, you can type `zrb` and press enter. ```bash zrb @@ -52,21 +58,36 @@ Commands: devtool Developer tools management env Environment variable management eval Evaluate Python expression - fibo fibo - hello hello - make Make things md5 MD5 operations explain Explain things project Project management - register-trainer register-trainer - start-server start-server - test-error test-error ubuntu Ubuntu related commands update Update zrb version Get Zrb version ``` -Once you find your task, you can just type the task without bothering about the parameters. Zrb will prompt you to fill the parameter interactively. +You can keep doing this until you find the task you want to execute + +```bash +zrb base64 +``` + +``` +Usage: zrb base64 [OPTIONS] COMMAND [ARGS]... + + Base64 operations + +Options: + --help Show this message and exit. + +Commands: + decode Decode base64 task + encode Encode base64 task +``` + +# Using prompt + +Once you find your task, you can just type the task without bothering about the parameters. Zrb will automatically prompt you to fill the parameter interactively. ```bash zrb base64 encode @@ -83,11 +104,13 @@ To run again: zrb base64 encode --text "non-credential-string" bm9uLWNyZWRlbnRpYWwtc3RyaW5n ``` +> __NOTE:__ To disable prompt, you can set `ZRB_SHOW_PROMPT` to `0` or `false`. Please refer to [configuration section](./configurations.md) for more information. + # Creating a project To make things more manageable, you can put related task definitions and resources under the same project. -You can create a project by invoking `zrb project create` as follow: +You can create a project under `my-project` directory by invoking the following command: ```bash zrb project create --project-dir my-project @@ -115,9 +138,11 @@ drwxr-xr-x 2 gofrendi gofrendi 4096 Jun 11 05:29 src -rw-r--r-- 1 gofrendi gofrendi 54 Jun 11 05:29 zrb_init.py ``` -A project is a directory containing `zrb_init.py`. All task definitions should be declared/imported to this file. +Every Zrb project contains a file named `zrb_init.py`. This file is your entry point to define all tasks/configurations. + +It is recommended that you define your tasks under `_automate` directory and import them into your `zrb_init.py`. This will help you manage the [separation of concerns](https://en.wikipedia.org/wiki/Separation_of_concerns). -When you create a project by using `zrb project create`, you will also see some other files/directory: +Aside from `zrb_init.py`, you will also find some other files/directory: - `.git` and `.gitignore`, indicating that your project is also a git repository. - `README.md`, your README file. @@ -127,7 +152,7 @@ When you create a project by using `zrb project create`, you will also see some - `_automate`, a directory contains task definitions that should be imported in `zrb_init.py`. - `src`, your project resources (e.g., source code, docker compose file, helm charts, etc) -By default, Zrb will create several tasks under your project. Try to type: +By default, Zrb will create a default `task-group` named `project`. Try to type: ```bash zrb project @@ -155,9 +180,9 @@ Commands: stop-containers Stop project containers ``` -# Adding a Cmd task +# Creating a simple task -Once your project has been created, it's time to add some tasks to your project. +Once your project has been created, you can add some new tasks to your project. Let's say you work for a company named `Arasaka`, and you want to show a cool CLI banner for your company. @@ -217,7 +242,7 @@ runner.register(show_banner) Cool. You make it. [Saburo Arasaka](https://cyberpunk.fandom.com/wiki/Saburo_Arasaka) will be proud of you 😉. -# Adding another Cmd Task: Run Jupyterlab +# Creating a long-running task Arasaka is a data-driven (and family-driven) company. They need their data scientists to experiment a lot to present the most valuable information/knowledge. @@ -369,15 +394,9 @@ Open up your browser on `http://localhost:8080` and start working. We have cover the minimum basics to work ~~for Arasaka~~ with Zrb. -No matter how complex your task will be, the flow will be similar: - -- You generate the task -- You modify the task -- You run the task - -To learn more about tasks and other concepts, you can visit [the concept section](concepts/README.md). +To learn more about tasks and other concepts, you can visit [Zrb concept section](concepts/README.md). -BTW, do you know that you can make and deploy a CRUD application without even touching your IDE/text editor? Check out [our tutorials](tutorials/README.md) for more cool tricks. +Also, do you know that you can make and deploy a CRUD application without even touching your IDE/text editor? Check out [our tutorials](tutorials/README.md) for more cool tricks. 🔖 [Table of Contents](README.md) \ No newline at end of file diff --git a/docs/for-maintainers.md b/docs/maintainer-guide.md similarity index 97% rename from docs/for-maintainers.md rename to docs/maintainer-guide.md index 4935081b..d2ca7ad7 100644 --- a/docs/for-maintainers.md +++ b/docs/maintainer-guide.md @@ -1,6 +1,6 @@ 🔖 [Table of Contents](README.md) -# For maintainers +# Maintainer Guide To publish Zrb, you need a `Pypi` account: diff --git a/docs/oops-i-did-it-again/README.md b/docs/oops-i-did-it-again/README.md new file mode 100644 index 00000000..29675633 --- /dev/null +++ b/docs/oops-i-did-it-again/README.md @@ -0,0 +1,10 @@ +🔖 [Table of Contents](../README.md) / [Troubleshooting](README.md) + +# Oops, I Did It Again: Most Common Mistakes When Working with Zrb + +Collection of common mistakes when working with Zrb + +- [Defining Different Tasks with The Same Name](defining-different-tasks-with-the-same-name.md) + + +🔖 [Table of Contents](../README.md) / [Troubleshooting](README.md) \ No newline at end of file diff --git a/docs/oops-i-did-it-again/defining-different-tasks-with-the-same-name.md b/docs/oops-i-did-it-again/defining-different-tasks-with-the-same-name.md new file mode 100644 index 00000000..061dfe2f --- /dev/null +++ b/docs/oops-i-did-it-again/defining-different-tasks-with-the-same-name.md @@ -0,0 +1,27 @@ +🔖 [Table of Contents](../README.md) / [Oops, I Did It Again](README.md) + +# Defining Different Tasks with The Same Name + +```python +from zrb import CmdTask, runner + +hello1 = CmdTask( + name='hello', + cmd='echo "hello mars"' +) +runner.register(hello1) + +hello2 = CmdTask( + name='hello', + cmd='echo "hello world"' +) +runner.register(hello2) +``` + +You can see that `hello1` and `hello2` share the same name. Thus, `hello2` will override `hello1` + +This leads to so many problems. + +For example, you believe that `zrb hello` should yield `hello mars`, yet it keep showing `hello world`. + +🔖 [Table of Contents](../README.md) / [Oops, I Did It Again](README.md) diff --git a/docs/tutorials/README.md b/docs/tutorials/README.md index dc42ea20..970bd0e9 100644 --- a/docs/tutorials/README.md +++ b/docs/tutorials/README.md @@ -7,5 +7,6 @@ - [Run task programmatically](run-task-programmatically.md) - [Define task dynamically](define-task-dynamically.md) - [Copy task](copy-task.md) +- [Extending CmdTask: Sending Message to Slack](extending-cmd-task.md) 🔖 [Table of Contents](../README.md) diff --git a/docs/tutorials/extending-cmd-task.md b/docs/tutorials/extending-cmd-task.md new file mode 100644 index 00000000..d73333dd --- /dev/null +++ b/docs/tutorials/extending-cmd-task.md @@ -0,0 +1,104 @@ +🔖 [Table of Contents](../README.md) / [Tutorials](README.md) + + +# Extending CmdTask: Sending Message to Slack + +```python +from typing import Any, Optional, Iterable, Union, Callable +from zrb import ( + runner, Group, AnyTask, CmdTask, AnyInput, StrInput, Env, EnvFile +) + +import jsons +import os + + +class SlackPrintTask(CmdTask): + + def __init__( + self, + name: str, + group: Optional[Group] = None, + inputs: Iterable[AnyInput] = [], + envs: Iterable[Env] = [], + env_files: Iterable[EnvFile] = [], + icon: Optional[str] = None, + color: Optional[str] = None, + description: str = '', + slack_channel_id: str = '', + slack_app_token: str = '', + message: str = '', + upstreams: Iterable[AnyTask] = [], + checkers: Iterable[AnyTask] = [], + checking_interval: Union[float, int] = 0, + retry: int = 2, + retry_interval: Union[float, int] = 1, + should_execute: Union[bool, str, Callable[..., bool]] = True, + ): + CmdTask.__init__( + self, + name=name, + group=group, + inputs=inputs, + envs=envs, + env_files=env_files, + icon=icon, + color=color, + description=description, + upstreams=upstreams, + checkers=checkers, + checking_interval=checking_interval, + retry=retry, + retry_interval=retry_interval, + should_execute=should_execute + ) + self._slack_channel_id = slack_channel_id + self._slack_app_token = slack_app_token + self._message = message + + def run(self, *args: Any, **kwargs: Any): + # Inject environment variables + self.inject_env_map( + env_map={ + 'CHANNEL_ID': self.render_str(self._slack_channel_id), + 'TOKEN': self.render_str(self._slack_app_token), + 'MESSAGE': self.render_str(self._message) + } + ) + return super().run(*args, **kwargs) + + def _get_cmd_str(self, *args: Any, **kwargs: Any): + # contruct json payload and replace all `"` with `\\"` + json_payload = jsons.dumps({ + 'channel': '$CHANNEL_ID', + 'blocks': [{ + 'type': 'section', + 'text': { + 'type': 'mrkdwn', + 'text': '$MESSAGE' + } + }] + }).replace('"', '\\"') + # send payload to slack API + return ' '.join([ + 'curl -H "Content-type: application/json"', + f'--data "{json_payload}"', + '-H "Authorization: Bearer $TOKEN"', + '-X POST https://slack.com/api/chat.postMessage' + ]) + + +say_hi = SlackPrintTask( + name='say-hi', + inputs=[ + StrInput(name='message', default='Hello world') + ], + slack_channel_id=os.getenv('SLACK_CHANNEL_ID'), + slack_app_token=os.getenv('SLACK_APP_TOKEN'), + message='{{ input.message }}', +) +runner.register(say_hi) + +``` + +🔖 [Table of Contents](../README.md) / [Tutorials](README.md) diff --git a/pyproject.toml b/pyproject.toml index 96abb89e..ba9f4c55 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "flit_core.buildapi" [project] name = "zrb" -version = "0.0.107" +version = "0.0.108" authors = [ { name="Go Frendi Gunawan", email="gofrendiasgard@gmail.com" }, ] diff --git a/src/zrb/action/runner.py b/src/zrb/action/runner.py index 2308a9cc..4da5c99b 100644 --- a/src/zrb/action/runner.py +++ b/src/zrb/action/runner.py @@ -27,7 +27,7 @@ def __init__(self, env_prefix: str = ''): logger.info(colored('Runner created', attrs=['dark'])) def register(self, task: AnyTask): - cmd_name = task.get_cmd_name() + cmd_name = task.get_complete_cmd_name() logger.debug(colored(f'Register task: {cmd_name}', attrs=['dark'])) self._tasks.append(task) task.set_has_cli_interface() diff --git a/src/zrb/task/any_task.py b/src/zrb/task/any_task.py index e3de9228..8185a148 100644 --- a/src/zrb/task/any_task.py +++ b/src/zrb/task/any_task.py @@ -130,6 +130,10 @@ def get_description(self) -> str: def get_cmd_name(self) -> str: pass + @abstractmethod + def get_complete_cmd_name(self) -> str: + pass + @abstractmethod def get_env_files(self) -> List[EnvFile]: pass diff --git a/src/zrb/task/base_task.py b/src/zrb/task/base_task.py index 1e0f661b..b4602c2c 100644 --- a/src/zrb/task/base_task.py +++ b/src/zrb/task/base_task.py @@ -21,7 +21,7 @@ from zrb.helper.string.modification import double_quote from zrb.helper.string.conversion import to_variable_name from zrb.helper.map.conversion import to_str as map_to_str -from zrb.config.config import show_advertisement +from zrb.config.config import show_advertisement, env_prefix import asyncio import copy @@ -333,6 +333,7 @@ async def _run_and_check_all( if raise_error: raise finally: + self._show_env_prefix() self._show_run_command() self._play_bell() @@ -372,6 +373,13 @@ async def _loop_check(self, show_done: bool = False) -> bool: await self.on_ready() return True + def _show_env_prefix(self): + if env_prefix == '': + return + colored_env_prefix = colored(env_prefix, color='yellow') + colored_label = colored('Your current environment: ', attrs=['dark']) + print(colored(f'{colored_label}{colored_env_prefix}'), file=sys.stderr) + def _show_run_command(self): params: List[str] = [double_quote(arg) for arg in self._args] for task_input in self.get_all_inputs(): @@ -381,14 +389,14 @@ def _show_run_command(self): kwarg_key = self._get_normalized_input_key(key) quoted_value = double_quote(str(self._kwargs[kwarg_key])) params.append(f'--{key} {quoted_value}') - run_cmd = self._get_complete_name() + run_cmd = self.get_complete_cmd_name() run_cmd_with_param = run_cmd if len(params) > 0: param_str = ' '.join(params) run_cmd_with_param += ' ' + param_str - colored_run_cmd = colored(f'{run_cmd_with_param}', color='yellow') + colored_command = colored(run_cmd_with_param, color='yellow') colored_label = colored('To run again: ', attrs=['dark']) - print(colored(f'{colored_label}{colored_run_cmd}'), file=sys.stderr) + print(colored(f'{colored_label}{colored_command}'), file=sys.stderr) async def _cached_check(self) -> bool: if self._is_check_triggered: diff --git a/src/zrb/task/base_task_composite.py b/src/zrb/task/base_task_composite.py index 7a5ac076..82764d23 100644 --- a/src/zrb/task/base_task_composite.py +++ b/src/zrb/task/base_task_composite.py @@ -462,11 +462,11 @@ def _get_common_prefix(self, show_time: bool) -> str: def _get_filled_complete_name(self) -> str: if self._filled_complete_name is not None: return self._filled_complete_name - complete_name = self._get_complete_name() + complete_name = self.get_complete_cmd_name() self._filled_complete_name = complete_name.rjust(LOG_NAME_LENGTH, ' ') return self._filled_complete_name - def _get_complete_name(self) -> str: + def get_complete_cmd_name(self) -> str: if self._complete_name is not None: return self._complete_name executable_prefix = '' diff --git a/src/zrb/task/cmd_task.py b/src/zrb/task/cmd_task.py index 6e49ad1e..756e6dd2 100644 --- a/src/zrb/task/cmd_task.py +++ b/src/zrb/task/cmd_task.py @@ -168,9 +168,9 @@ def _set_cwd( self, cwd: Optional[Union[str, pathlib.Path]] ): if cwd is None: - self.cwd: Union[str, pathlib.Path] = os.getcwd() + self._cwd: Union[str, pathlib.Path] = os.getcwd() return - self.cwd: Union[str, pathlib.Path] = cwd + self._cwd: Union[str, pathlib.Path] = os.path.abspath(cwd) def to_function( self, env_prefix: str = '', raise_error: bool = True @@ -195,12 +195,12 @@ async def run(self, *args: Any, **kwargs: Any) -> CmdResult: cmd = self._get_cmd_str(*args, **kwargs) env_map = self._get_shell_env_map() self.print_out_dark('Run script: ' + self._get_multiline_repr(cmd)) - self.print_out_dark('Working directory: ' + self.cwd) + self.print_out_dark('Working directory: ' + self._cwd) self._output_buffer = [] self._error_buffer = [] process = await asyncio.create_subprocess_shell( cmd, - cwd=self.cwd, + cwd=self._cwd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, env=env_map, diff --git a/src/zrb/task/docker_compose_task.py b/src/zrb/task/docker_compose_task.py index 8081f2dd..b1ccf766 100644 --- a/src/zrb/task/docker_compose_task.py +++ b/src/zrb/task/docker_compose_task.py @@ -150,11 +150,9 @@ def __init__( self._compose_runtime_file = self._get_compose_runtime_file( self._compose_template_file ) - # append services env and env_files to current task - for _, service_config in self._compose_service_configs.items(): - self._env_files += service_config.get_env_files() - self._envs += service_config.get_envs() - self._add_compose_envs() + # Flag to make mark whether service config and compose environments + # has been added to this task's envs and env_files + self._is_additional_env_added = False def copy(self) -> TDockerComposeTask: return super().copy() @@ -167,6 +165,46 @@ async def run(self, *args, **kwargs: Any) -> CmdResult: os.remove(self._compose_runtime_file) return result + def _get_all_envs(self) -> Mapping[str, Env]: + ''' + This method override BaseTask's _get_all_envs. + Whenever _get_all_envs is called, we want to make sure that: + - Service config's envs and env_files are included + - Any environment defined in docker compose file is also included + ''' + if self._is_additional_env_added: + return super()._get_all_envs() + self._is_additional_env_added = True + # define additional envs and additonal env_files + additional_envs: List[Env] = [] + additional_env_files: List[EnvFile] = [] + # populate additional envs and additional env_files + # with service configs + for _, service_config in self._compose_service_configs.items(): + additional_env_files += service_config.get_env_files() + additional_envs += service_config.get_envs() + # populate additional envs and additional env_files with + # compose envs + data = read_compose_file(self._compose_template_file) + env_map = fetch_compose_file_env_map(data) + registered_env_map: Mapping[str, bool] = {} + for key, value in env_map.items(): + # Need to get this everytime because we only want + # the first compose file env value for a certain key + if key in registered_env_map: + continue + os_name = key + if self._compose_env_prefix != '': + os_name = f'{self._compose_env_prefix}_{os_name}' + compose_env = Env(name=key, os_name=os_name, default=value) + additional_envs.append(compose_env) + registered_env_map[key] = True + # Add additional envs and addition env files to this task + self._envs = additional_envs + list(self._envs) + self._env_files = additional_env_files + list(self._env_files) + # get all envs + return super()._get_all_envs() + def _generate_compose_runtime_file(self): compose_data = read_compose_file(self._compose_template_file) for service, service_config in self._compose_service_configs.items(): @@ -175,10 +213,10 @@ def _generate_compose_runtime_file(self): for env_file in env_files: envs += env_file.get_envs() envs += service_config.get_envs() - compose_data = self._add_service_env(compose_data, service, envs) + compose_data = self._apply_service_env(compose_data, service, envs) write_compose_file(self._compose_runtime_file, compose_data) - def _add_service_env( + def _apply_service_env( self, compose_data: Any, service: str, envs: List[Env] ) -> Any: # service not found @@ -237,32 +275,6 @@ def _get_service_new_env_list( def _get_env_compose_value(self, env: Env) -> str: return '${' + env.name + ':-' + env.default + '}' - def _add_compose_envs(self): - data = read_compose_file(self._compose_template_file) - env_map = fetch_compose_file_env_map(data) - for key, value in env_map.items(): - # Need to get this everytime because we only want - # the first compose file env value for a certain key - existing_env_map = self._get_existing_env_map() - if key in existing_env_map: - continue - os_name = key - if self._compose_env_prefix != '': - os_name = f'{self._compose_env_prefix}_{os_name}' - self.add_envs(Env(name=key, os_name=os_name, default=value)) - - def _get_existing_env_map(self) -> Mapping[str, str]: - env_map: Mapping[str, str] = {} - for env_file in self._env_files: - envs = env_file.get_envs() - env_map.update({ - env.name: env.default for env in envs - }) - env_map.update({ - env.name: env.default for env in self._envs - }) - return env_map - def _get_compose_runtime_file(self, compose_file_name: str) -> str: directory, file = os.path.split(compose_file_name) prefix = '_' if file.startswith('.') else '._' @@ -288,14 +300,14 @@ def _get_compose_template_file(self, compose_file: Optional[str]) -> str: 'compose.yml', 'compose.yaml', 'docker-compose.yml', 'docker-compose.yaml' ]: - if os.path.exists(os.path.join(self.cwd, _compose_file)): - return os.path.join(self.cwd, _compose_file) + if os.path.exists(os.path.join(self._cwd, _compose_file)): + return os.path.join(self._cwd, _compose_file) return - raise Exception(f'Cannot find compose file on {self.cwd}') + raise Exception(f'Cannot find compose file on {self._cwd}') if os.path.isabs(compose_file) and os.path.exists(compose_file): return compose_file - if os.path.exists(os.path.join(self.cwd, compose_file)): - return os.path.join(self.cwd, compose_file) + if os.path.exists(os.path.join(self._cwd, compose_file)): + return os.path.join(self._cwd, compose_file) raise Exception(f'Invalid compose file: {compose_file}') def _get_cmd_str(self, *args: Any, **kwargs: Any) -> str: diff --git a/test/task/docker_compose_task/test_docker_compose_task.py b/test/task/docker_compose_task/test_docker_compose_task.py index 2df479f5..07ab5aed 100644 --- a/test/task/docker_compose_task/test_docker_compose_task.py +++ b/test/task/docker_compose_task/test_docker_compose_task.py @@ -1,4 +1,6 @@ -from zrb.task.docker_compose_task import DockerComposeTask, ServiceConfig, Env, EnvFile +from zrb.task.docker_compose_task import ( + DockerComposeTask, ServiceConfig, Env, EnvFile +) import pathlib import os