From 27386d9414af1a87e4ad21e52a36b9d8bc06a480 Mon Sep 17 00:00:00 2001 From: OjusWiZard Date: Tue, 15 Oct 2024 20:06:56 +0530 Subject: [PATCH] Feat (deploy): All multiple services on docker Signed-off-by: OjusWiZard --- autonomy/cli/deploy.py | 84 +++++++++------ autonomy/cli/helpers/deployment.py | 15 +-- autonomy/cli/replay.py | 2 +- autonomy/deploy/base.py | 20 +++- autonomy/deploy/build.py | 4 + .../deploy/generators/docker_compose/base.py | 61 ++++++++++- docs/api/cli/deploy.md | 8 ++ docs/api/cli/helpers/deployment.md | 14 +-- docs/api/deploy/base.md | 18 +++- docs/api/deploy/build.md | 4 +- .../deploy/generators/docker_compose/base.md | 4 +- .../test_deploy/test_build/test_deployment.py | 100 +++++++++++++++--- .../test_cli/test_replay/test_agent.py | 7 +- .../test_cli/test_replay/test_tendermint.py | 2 +- 14 files changed, 265 insertions(+), 78 deletions(-) diff --git a/autonomy/cli/deploy.py b/autonomy/cli/deploy.py index 0f90c84bfb..f4d54d314b 100644 --- a/autonomy/cli/deploy.py +++ b/autonomy/cli/deploy.py @@ -19,6 +19,7 @@ """Deploy CLI module.""" +import os import shutil from pathlib import Path from typing import Optional, cast @@ -38,7 +39,6 @@ from autonomy.cli.helpers.deployment import ( build_and_deploy_from_token, build_deployment, - build_hash_id, run_deployment, stop_deployment, ) @@ -55,6 +55,7 @@ DEFAULT_AGENT_MEMORY_LIMIT, DEFAULT_AGENT_MEMORY_REQUEST, NotValidKeysFile, + build_hash_id, ) from autonomy.deploy.constants import INFO, LOGGING_LEVELS from autonomy.deploy.generators.docker_compose.base import DockerComposeGenerator @@ -122,6 +123,13 @@ def deploy_group( default=None, help="Number of agents.", ) +@click.option( + "--number-of-services", + "number_of_services", + type=int, + default=1, + help="Number of services.", +) @click.option( "--docker", "deployment_type", @@ -219,6 +227,7 @@ def build_deployment_command( # pylint: disable=too-many-arguments, too-many-lo dev_mode: bool, registry: str, number_of_agents: Optional[int] = None, + number_of_services: int = 1, password: Optional[str] = None, open_aea_dir: Optional[Path] = None, packages_dir: Optional[Path] = None, @@ -246,9 +255,6 @@ def build_deployment_command( # pylint: disable=too-many-arguments, too-many-lo message = f"No such file or directory: {keys_file}. Please provide valid path for keys file." raise click.ClickException(message) - build_dir = Path( - output_dir or DEFAULT_BUILD_FOLDER.format(build_hash_id()) - ).absolute() if dev_mode: packages_dir = _validate_packages_path(path=packages_dir) open_aea_dir = _validate_open_aea_dir(path=open_aea_dir) @@ -256,35 +262,47 @@ def build_deployment_command( # pylint: disable=too-many-arguments, too-many-lo ctx = cast(Context, click_context.obj) ctx.registry_type = registry - try: - build_deployment( - keys_file=keys_file, - build_dir=build_dir, - deployment_type=deployment_type, - dev_mode=dev_mode, - number_of_agents=number_of_agents, - packages_dir=packages_dir, - open_aea_dir=open_aea_dir, - log_level=log_level, - apply_environment_variables=aev, - image_version=image_version, - use_hardhat=use_hardhat, - use_acn=use_acn, - use_tm_testnet_setup=use_tm_testnet_setup, - image_author=image_author, - resources={ - "agent": { - "limit": {"cpu": agent_cpu_limit, "memory": agent_memory_limit}, - "requested": { - "cpu": agent_cpu_request, - "memory": agent_memory_request, - }, - } - }, - ) - except (NotValidKeysFile, FileNotFoundError, FileExistsError) as e: - shutil.rmtree(build_dir) - raise click.ClickException(str(e)) from e + abci_build_count = len( + [name for name in os.listdir() if name.startswith("abci_build_")] + ) + + for service_index in range(number_of_services): + service_hash_id = build_hash_id() + build_dir = Path( + output_dir or DEFAULT_BUILD_FOLDER.format(service_hash_id) + ).absolute() + + try: + build_deployment( + service_hash_id=service_hash_id, + service_offset=abci_build_count + service_index, + keys_file=keys_file, + build_dir=build_dir, + deployment_type=deployment_type, + dev_mode=dev_mode, + number_of_agents=number_of_agents, + packages_dir=packages_dir, + open_aea_dir=open_aea_dir, + log_level=log_level, + apply_environment_variables=aev, + image_version=image_version, + use_hardhat=use_hardhat, + use_acn=use_acn, + use_tm_testnet_setup=use_tm_testnet_setup, + image_author=image_author, + resources={ + "agent": { + "limit": {"cpu": agent_cpu_limit, "memory": agent_memory_limit}, + "requested": { + "cpu": agent_cpu_request, + "memory": agent_memory_request, + }, + } + }, + ) + except (NotValidKeysFile, FileNotFoundError, FileExistsError) as e: + shutil.rmtree(build_dir) + raise click.ClickException(str(e)) from e @deploy_group.command(name="run") diff --git a/autonomy/cli/helpers/deployment.py b/autonomy/cli/helpers/deployment.py index 78c9eb2159..e95ac464d7 100644 --- a/autonomy/cli/helpers/deployment.py +++ b/autonomy/cli/helpers/deployment.py @@ -21,7 +21,6 @@ import os import shutil import time -from base64 import urlsafe_b64encode from pathlib import Path from typing import Dict, List, Optional, Tuple @@ -42,7 +41,7 @@ from autonomy.configurations.constants import DEFAULT_SERVICE_CONFIG_FILE from autonomy.configurations.loader import load_service_config from autonomy.constants import DEFAULT_BUILD_FOLDER -from autonomy.deploy.base import Resources +from autonomy.deploy.base import Resources, build_hash_id from autonomy.deploy.build import generate_deployment from autonomy.deploy.constants import ( AGENT_KEYS_DIR, @@ -113,11 +112,6 @@ def _load_compose_project(build_dir: Path) -> Project: raise -def build_hash_id() -> str: - """Generate a random 4 character hash id for the deployment build directory name.""" - return urlsafe_b64encode(bytes(os.urandom(3))).decode() - - def run_deployment( build_dir: Path, no_recreate: bool = False, @@ -200,6 +194,8 @@ def build_deployment( # pylint: disable=too-many-arguments, too-many-locals use_tm_testnet_setup: bool = False, image_author: Optional[str] = None, resources: Optional[Resources] = None, + service_hash_id: Optional[str] = None, + service_offset: int = 0, ) -> None: """Build deployment.""" @@ -215,7 +211,12 @@ def build_deployment( # pylint: disable=too-many-arguments, too-many-locals build_dir.mkdir() _build_dirs(build_dir) + if service_hash_id is None: + service_hash_id = build_hash_id() + report = generate_deployment( + service_hash_id=service_hash_id, + service_offset=service_offset, service_path=Path.cwd(), type_of_deployment=deployment_type, keys_file=keys_file, diff --git a/autonomy/cli/replay.py b/autonomy/cli/replay.py index c9dc350cdb..6a80b86a9d 100644 --- a/autonomy/cli/replay.py +++ b/autonomy/cli/replay.py @@ -26,8 +26,8 @@ from aea.cli.utils.click_utils import reraise_as_click_exception from aea.configurations.constants import PACKAGES -from autonomy.cli.helpers.deployment import build_hash_id from autonomy.constants import DEFAULT_BUILD_FOLDER, DOCKER_COMPOSE_YAML +from autonomy.deploy.base import build_hash_id from autonomy.deploy.constants import PERSISTENT_DATA_DIR, TM_STATE_DIR from autonomy.replay.agent import AgentRunner from autonomy.replay.tendermint import build_tendermint_apps diff --git a/autonomy/deploy/base.py b/autonomy/deploy/base.py index 2515077851..44adc93c85 100644 --- a/autonomy/deploy/base.py +++ b/autonomy/deploy/base.py @@ -22,6 +22,7 @@ import json import logging import os +from base64 import urlsafe_b64encode from copy import copy, deepcopy from pathlib import Path from typing import Any, Dict, List, Optional, Set, Tuple, Union, cast @@ -144,6 +145,11 @@ class Resources(TypedDict): ) +def build_hash_id() -> str: + """Generate a random 4 character hash id for the deployment build directory name.""" + return urlsafe_b64encode(bytes(os.urandom(3))).decode() + + class ServiceBuilder: # pylint: disable=too-many-instance-attributes """Class to assist with generating deployments.""" @@ -156,6 +162,8 @@ def __init__( # pylint: disable=too-many-arguments keys: Optional[List[Union[List[Dict[str, str]], Dict[str, str]]]] = None, agent_instances: Optional[List[str]] = None, apply_environment_variables: bool = False, + service_hash_id: Optional[str] = None, + service_offset: int = 0, ) -> None: """Initialize the Base Deployment.""" if apply_environment_variables: @@ -167,8 +175,14 @@ def __init__( # pylint: disable=too-many-arguments ) self.service = service + self.service_hash_id = ( + build_hash_id() if service_hash_id is None else service_hash_id + ) + self.service_offset = service_offset - self._service_name_clean = self.service.name.replace("_", "") + self._service_name_clean = ( + self.service.name.replace("_", "") + self.service_hash_id + ) self._keys = keys or [] self._agent_instances = agent_instances self._all_participants = self.try_get_all_participants() @@ -254,6 +268,8 @@ def from_dir( # pylint: disable=too-many-arguments agent_instances: Optional[List[str]] = None, apply_environment_variables: bool = False, dev_mode: bool = False, + service_hash_id: Optional[str] = None, + service_offset: int = 0, ) -> "ServiceBuilder": """Service builder from path.""" service = load_service_config( @@ -264,6 +280,8 @@ def from_dir( # pylint: disable=too-many-arguments service_builder = cls( service=service, + service_hash_id=service_hash_id, + service_offset=service_offset, apply_environment_variables=apply_environment_variables, ) diff --git a/autonomy/deploy/build.py b/autonomy/deploy/build.py index ea1a1d2d8c..df65f13d9e 100644 --- a/autonomy/deploy/build.py +++ b/autonomy/deploy/build.py @@ -33,6 +33,8 @@ def generate_deployment( # pylint: disable=too-many-arguments, too-many-locals + service_hash_id: str, + service_offset: int, type_of_deployment: str, keys_file: Path, service_path: Path, @@ -66,6 +68,8 @@ def generate_deployment( # pylint: disable=too-many-arguments, too-many-locals agent_instances=agent_instances, apply_environment_variables=apply_environment_variables, dev_mode=dev_mode, + service_hash_id=service_hash_id, + service_offset=service_offset, ) service_builder.deplopyment_type = type_of_deployment service_builder.log_level = log_level diff --git a/autonomy/deploy/generators/docker_compose/base.py b/autonomy/deploy/generators/docker_compose/base.py index 21ab949ea2..28e4bbcf0e 100644 --- a/autonomy/deploy/generators/docker_compose/base.py +++ b/autonomy/deploy/generators/docker_compose/base.py @@ -21,8 +21,9 @@ import ipaddress import os from pathlib import Path -from typing import Dict, List, Optional, cast +from typing import Dict, List, Optional, Set, cast +import yaml from aea.configurations.constants import ( DEFAULT_LEDGER, LEDGER, @@ -33,6 +34,7 @@ from docker.constants import DEFAULT_NPIPE, IS_WINDOWS_PLATFORM from docker.errors import DockerException +from autonomy.configurations.constants import DEFAULT_SERVICE_CONFIG_FILE from autonomy.constants import ( ACN_IMAGE_NAME, ACN_IMAGE_VERSION, @@ -192,10 +194,12 @@ def __init__( self, name: str, base: ipaddress.IPv4Network = BASE_SUBNET, + used_subnets: Optional[Set[str]] = None, ) -> None: """Initialize.""" self.name = name self.base = base + self.used_subnets = set() if used_subnets is None else used_subnets self.subnet = self.build() self._address_offeset = NETWORK_ADDRESS_OFFSET @@ -214,7 +218,7 @@ def build( ) -> ipaddress.IPv4Network: """Initialize network params.""" docker = get_docker_client() - used_subnets = set() + used_subnets = self.used_subnets for network in docker.networks.list(): # Network already exists if f"abci_build_{self.name}" == network.attrs["Name"]: @@ -245,6 +249,55 @@ class DockerComposeGenerator(BaseDeploymentGenerator): output_name: str = DOCKER_COMPOSE_YAML deployment_type: str = "docker-compose" + def _find_occupied_networks(self, network_name: str) -> Set[str]: + """Find occupied networks on the host system.""" + used_ports: Dict[str, str] = {} # find occupied ports + for config_name in ("agent", "tendermint"): + for port_mapping in ( + self.service_builder.service.deployment_config.get(config_name, {}) + .get("ports", {}) + .values() + ): + for host_port, _ in port_mapping.items(): + if host_port in used_ports: + print( + f"WARNING: Port {host_port} is already used by {used_ports[host_port]}." + ) + else: + used_ports[str(host_port)] = DEFAULT_SERVICE_CONFIG_FILE + + used_networks = set() # find occupied networks + for build in Path.cwd().glob("abci_build_*"): + if not (build / DOCKER_COMPOSE_YAML).exists(): + continue + + compose_config = yaml.safe_load((build / DOCKER_COMPOSE_YAML).read_text()) + for subnet in ( + compose_config.get("networks", {}) + .get(network_name, {}) + .get("ipam", {}) + .get("config", []) + ): + used_networks.add(subnet["subnet"]) + + services = compose_config.get("services", {}) + for name, service in services.items(): + for port_mapping_str in service.get("ports", []): + port_mapping = port_mapping_str.split(":") + if len(port_mapping) != 2: + continue + + host_port = port_mapping[0] + if host_port in used_ports: + print( + f"WARNING: Port {host_port} is already used by {used_ports[host_port]}." + f" Please adjust the port in {build / DOCKER_COMPOSE_YAML} manually.", + ) + else: + used_ports[host_port] = name + + return used_networks + def generate_config_tendermint(self) -> "DockerComposeGenerator": """Generate the command to configure tendermint testnet.""" if not self.use_tm_testnet_setup: @@ -280,9 +333,9 @@ def generate( use_acn: bool = False, ) -> "DockerComposeGenerator": """Generate the new configuration.""" - network_name = f"service_{self.service_builder.service.name}_localnet" - network = Network(name=network_name) + used_networks = self._find_occupied_networks(network_name) + network = Network(name=network_name, used_subnets=used_networks) image_version = image_version or self.service_builder.service.agent.hash if self.dev_mode: runtime_image = DEVELOPMENT_IMAGE diff --git a/docs/api/cli/deploy.md b/docs/api/cli/deploy.md index ece74159b8..5635726c15 100644 --- a/docs/api/cli/deploy.md +++ b/docs/api/cli/deploy.md @@ -46,6 +46,13 @@ Deploy an agent service. default=None, help="Number of agents.", ) +@click.option( + "--number-of-services", + "number_of_services", + type=int, + default=1, + help="Number of services.", +) @click.option( "--docker", "deployment_type", @@ -145,6 +152,7 @@ def build_deployment_command( dev_mode: bool, registry: str, number_of_agents: Optional[int] = None, + number_of_services: int = 1, password: Optional[str] = None, open_aea_dir: Optional[Path] = None, packages_dir: Optional[Path] = None, diff --git a/docs/api/cli/helpers/deployment.md b/docs/api/cli/helpers/deployment.md index c99bb36dfa..116d2c0164 100644 --- a/docs/api/cli/helpers/deployment.md +++ b/docs/api/cli/helpers/deployment.md @@ -4,16 +4,6 @@ Deployment helpers. - - -#### build`_`hash`_`id - -```python -def build_hash_id() -> str -``` - -Generate a random 4 character hash id for the deployment build directory name. - #### run`_`deployment @@ -59,7 +49,9 @@ def build_deployment(keys_file: Path, use_acn: bool = False, use_tm_testnet_setup: bool = False, image_author: Optional[str] = None, - resources: Optional[Resources] = None) -> None + resources: Optional[Resources] = None, + service_hash_id: Optional[str] = None, + service_offset: int = 0) -> None ``` Build deployment. diff --git a/docs/api/deploy/base.md b/docs/api/deploy/base.md index 0250f4fac4..ba7113c68f 100644 --- a/docs/api/deploy/base.md +++ b/docs/api/deploy/base.md @@ -66,6 +66,16 @@ class Resources(TypedDict) Deployment resources. + + +#### build`_`hash`_`id + +```python +def build_hash_id() -> str +``` + +Generate a random 4 character hash id for the deployment build directory name. + ## ServiceBuilder Objects @@ -85,7 +95,9 @@ def __init__(service: Service, keys: Optional[List[Union[List[Dict[str, str]], Dict[str, str]]]] = None, agent_instances: Optional[List[str]] = None, - apply_environment_variables: bool = False) -> None + apply_environment_variables: bool = False, + service_hash_id: Optional[str] = None, + service_offset: int = 0) -> None ``` Initialize the Base Deployment. @@ -165,7 +177,9 @@ def from_dir(cls, number_of_agents: Optional[int] = None, agent_instances: Optional[List[str]] = None, apply_environment_variables: bool = False, - dev_mode: bool = False) -> "ServiceBuilder" + dev_mode: bool = False, + service_hash_id: Optional[str] = None, + service_offset: int = 0) -> "ServiceBuilder" ``` Service builder from path. diff --git a/docs/api/deploy/build.md b/docs/api/deploy/build.md index ab174098cd..4624340171 100644 --- a/docs/api/deploy/build.md +++ b/docs/api/deploy/build.md @@ -9,7 +9,9 @@ Script for generating deployment environments. #### generate`_`deployment ```python -def generate_deployment(type_of_deployment: str, +def generate_deployment(service_hash_id: str, + service_offset: int, + type_of_deployment: str, keys_file: Path, service_path: Path, build_dir: Path, diff --git a/docs/api/deploy/generators/docker_compose/base.md b/docs/api/deploy/generators/docker_compose/base.md index 3ee84ab3c2..71cb562363 100644 --- a/docs/api/deploy/generators/docker_compose/base.md +++ b/docs/api/deploy/generators/docker_compose/base.md @@ -68,7 +68,9 @@ Class to represent network of the service. #### `__`init`__` ```python -def __init__(name: str, base: ipaddress.IPv4Network = BASE_SUBNET) -> None +def __init__(name: str, + base: ipaddress.IPv4Network = BASE_SUBNET, + used_subnets: Optional[Set[str]] = None) -> None ``` Initialize. diff --git a/tests/test_autonomy/test_cli/test_deploy/test_build/test_deployment.py b/tests/test_autonomy/test_cli/test_deploy/test_build/test_deployment.py index 90d0b7b42e..3420891da0 100644 --- a/tests/test_autonomy/test_cli/test_deploy/test_build/test_deployment.py +++ b/tests/test_autonomy/test_cli/test_deploy/test_build/test_deployment.py @@ -22,8 +22,9 @@ import json import os import shutil +from itertools import cycle from pathlib import Path -from typing import Dict, List, Tuple +from typing import Dict, List, Optional, Tuple from unittest import mock import yaml @@ -34,14 +35,18 @@ ETHEREUM_ENCRYPTION_PASSWORD, ) -from autonomy.cli.helpers.deployment import build_hash_id -from autonomy.constants import DEFAULT_BUILD_FOLDER, DEFAULT_DOCKER_IMAGE_AUTHOR +from autonomy.constants import ( + DEFAULT_BUILD_FOLDER, + DEFAULT_DOCKER_IMAGE_AUTHOR, + DOCKER_COMPOSE_YAML, +) from autonomy.deploy.base import ( DEFAULT_AGENT_CPU_LIMIT, DEFAULT_AGENT_CPU_REQUEST, DEFAULT_AGENT_MEMORY_LIMIT, DEFAULT_AGENT_MEMORY_REQUEST, ServiceBuilder, + build_hash_id, ) from autonomy.deploy.constants import ( DEBUG, @@ -85,8 +90,7 @@ def setup(self) -> None: ) with OS_ENV_PATCH: self.spec = ServiceBuilder.from_dir( - self.t / "register_reset", - self.keys_file, + self.t / "register_reset", self.keys_file, service_hash_id="test" ) os.chdir(self.t / "register_reset") @@ -111,11 +115,20 @@ def check_kubernetes_build(cls, build_dir: Path) -> None: def load_and_check_docker_compose_file( self, path: Path, + service_hash_id: Optional[str] = None, ) -> Dict: """Load docker compose config.""" with open(path, "r", encoding="utf-8") as fp: docker_compose = yaml.safe_load(fp) + if service_hash_id is not None: + with OS_ENV_PATCH: + self.spec = ServiceBuilder.from_dir( + self.t / "register_reset", + self.keys_file, + service_hash_id=service_hash_id, + ) + assert any( [key in docker_compose for key in ["version", "services", "networks"]] ) @@ -159,7 +172,9 @@ def test_docker_compose_build( """Run tests.""" build_dir = self.t / DEFAULT_BUILD_FOLDER.format(build_hash_id()) - with mock.patch("os.chown"), OS_ENV_PATCH: + with mock.patch("os.chown"), OS_ENV_PATCH, mock.patch( + "autonomy.cli.deploy.build_hash_id", return_value="test" + ): result = self.run_cli( ( str(self.keys_file), @@ -187,7 +202,9 @@ def test_docker_compose_build_with_testnet( """Run tests.""" build_dir = self.t / DEFAULT_BUILD_FOLDER.format(build_hash_id()) - with mock.patch("os.chown"), OS_ENV_PATCH: + with mock.patch("os.chown"), OS_ENV_PATCH, mock.patch( + "autonomy.cli.deploy.build_hash_id", return_value="test" + ): result = self.run_cli( ( str(self.keys_file), @@ -207,7 +224,9 @@ def test_docker_compose_build_log_level( """Run tests.""" build_dir = self.t / DEFAULT_BUILD_FOLDER.format(build_hash_id()) - with mock.patch("os.chown"), OS_ENV_PATCH: + with mock.patch("os.chown"), OS_ENV_PATCH, mock.patch( + "autonomy.cli.deploy.build_hash_id", return_value="test" + ): result = self.run_cli( ( str(self.keys_file), @@ -249,7 +268,9 @@ def test_docker_compose_build_dev( """Run tests.""" build_dir = self.t / DEFAULT_BUILD_FOLDER.format(build_hash_id()) - with mock.patch("os.chown"), OS_ENV_PATCH: + with mock.patch("os.chown"), OS_ENV_PATCH, mock.patch( + "autonomy.cli.deploy.build_hash_id", return_value="test" + ): result = self.run_cli( ( str(self.keys_file), @@ -301,7 +322,9 @@ def test_docker_compose_password( keys_file = Path(ETHEREUM_ENCRYPTED_KEYS) build_dir = self.t / DEFAULT_BUILD_FOLDER.format(build_hash_id()) - with mock.patch("os.chown"), OS_ENV_PATCH: + with mock.patch("os.chown"), OS_ENV_PATCH, mock.patch( + "autonomy.cli.deploy.build_hash_id", return_value="test" + ): result = self.run_cli( ( str(keys_file), @@ -352,7 +375,9 @@ def test_include_acn_and_hardhat_nodes( """Run tests.""" build_dir = self.t / DEFAULT_BUILD_FOLDER.format(build_hash_id()) - with mock.patch("os.chown"), OS_ENV_PATCH: + with mock.patch("os.chown"), OS_ENV_PATCH, mock.patch( + "autonomy.cli.deploy.build_hash_id", return_value="test" + ): result = self.run_cli( ( str(self.keys_file), @@ -383,7 +408,9 @@ def test_build_dev_failures( ) -> None: """Run tests.""" - with mock.patch("os.chown"), OS_ENV_PATCH: + with mock.patch("os.chown"), OS_ENV_PATCH, mock.patch( + "autonomy.cli.deploy.build_hash_id", return_value="test" + ): result = self.run_cli( ( str(self.keys_file), @@ -424,7 +451,9 @@ def test_docker_compose_build_image_author_flag_default( """Run tests.""" build_dir = self.t / DEFAULT_BUILD_FOLDER.format(build_hash_id()) - with mock.patch("os.chown"), OS_ENV_PATCH: + with mock.patch("os.chown"), OS_ENV_PATCH, mock.patch( + "autonomy.cli.deploy.build_hash_id", return_value="test" + ): result = self.run_cli( ( str(self.keys_file), @@ -462,7 +491,9 @@ def test_docker_compose_build_image_author_flag_custom( build_dir = self.t / DEFAULT_BUILD_FOLDER.format(build_hash_id()) image_author = "some_author" - with mock.patch("os.chown"), OS_ENV_PATCH: + with mock.patch("os.chown"), OS_ENV_PATCH, mock.patch( + "autonomy.cli.deploy.build_hash_id", return_value="test" + ): result = self.run_cli( ( str(self.keys_file), @@ -494,6 +525,47 @@ def test_docker_compose_build_image_author_flag_custom( == image_author ) + def test_docker_compose_build_multiple( + self, + ) -> None: + """Run tests.""" + + mock_build_hash_id_cycle = cycle(["test", "test2"]) + + def mock_build_hash_id() -> str: + """Mock build hash id.""" + return next(mock_build_hash_id_cycle) + + with mock.patch("os.chown"), OS_ENV_PATCH, mock.patch( + "autonomy.cli.deploy.build_hash_id", new=mock_build_hash_id + ): + result = self.run_cli( + ( + str(self.keys_file), + "--number-of-services", + "2", + ) + ) + + assert result.exit_code == 0, result.output + + builds_count = 0 + for build_dir in Path.cwd().glob("abci_build_*"): + if not (build_dir / DOCKER_COMPOSE_YAML).exists(): + continue + + builds_count += 1 + + self.check_docker_compose_build( + build_dir=build_dir, + ) + self.load_and_check_docker_compose_file( + path=build_dir / DockerComposeGenerator.output_name, + service_hash_id=mock_build_hash_id(), + ) + + assert builds_count == 2 + class TestKubernetesBuild(BaseDeployBuildTest): """Test kubernetes builds.""" diff --git a/tests/test_autonomy/test_cli/test_replay/test_agent.py b/tests/test_autonomy/test_cli/test_replay/test_agent.py index 98223fcb20..560c0c7935 100644 --- a/tests/test_autonomy/test_cli/test_replay/test_agent.py +++ b/tests/test_autonomy/test_cli/test_replay/test_agent.py @@ -27,9 +27,12 @@ from unittest import mock from autonomy.cli import cli -from autonomy.cli.helpers.deployment import build_hash_id from autonomy.constants import DEFAULT_BUILD_FOLDER -from autonomy.deploy.base import TENDERMINT_COM_URL_PARAM, TENDERMINT_URL_PARAM +from autonomy.deploy.base import ( + TENDERMINT_COM_URL_PARAM, + TENDERMINT_URL_PARAM, + build_hash_id, +) from autonomy.replay.agent import AgentRunner from tests.conftest import ROOT_DIR, skip_docker_tests diff --git a/tests/test_autonomy/test_cli/test_replay/test_tendermint.py b/tests/test_autonomy/test_cli/test_replay/test_tendermint.py index 83593ce915..b5fc81045e 100644 --- a/tests/test_autonomy/test_cli/test_replay/test_tendermint.py +++ b/tests/test_autonomy/test_cli/test_replay/test_tendermint.py @@ -30,8 +30,8 @@ import flask from autonomy.cli import cli -from autonomy.cli.helpers.deployment import build_hash_id from autonomy.constants import DEFAULT_BUILD_FOLDER +from autonomy.deploy.base import build_hash_id from autonomy.deploy.constants import PERSISTENT_DATA_DIR, TM_STATE_DIR from autonomy.replay.tendermint import TendermintNetwork