Merge pull request #2267 from OjusWiZard/fix/multi-service

Feat (deploy): Allow multiple services on docker

autonomy/cli/deploy.py

@@ -19,6 +19,7 @@
 
 """Deploy CLI module."""
 
+import os
 import shutil
 from pathlib import Path
 from typing import Optional, cast
@@ -38,7 +39,6 @@
 from autonomy.cli.helpers.deployment import (
     build_and_deploy_from_token,
     build_deployment,
-    build_hash_id,
     run_deployment,
     stop_deployment,
 )
@@ -55,6 +55,7 @@
     DEFAULT_AGENT_MEMORY_LIMIT,
     DEFAULT_AGENT_MEMORY_REQUEST,
     NotValidKeysFile,
+    build_hash_id,
 )
 from autonomy.deploy.constants import INFO, LOGGING_LEVELS
 from autonomy.deploy.generators.docker_compose.base import DockerComposeGenerator
@@ -122,6 +123,13 @@ def deploy_group(
     default=None,
     help="Number of agents.",
 )
+@click.option(
+    "--number-of-services",
+    "number_of_services",
+    type=int,
+    default=1,
+    help="Number of services.",
+)
 @click.option(
     "--docker",
     "deployment_type",
@@ -219,6 +227,7 @@ def build_deployment_command(  # pylint: disable=too-many-arguments, too-many-lo
     dev_mode: bool,
     registry: str,
     number_of_agents: Optional[int] = None,
+    number_of_services: int = 1,
     password: Optional[str] = None,
     open_aea_dir: Optional[Path] = None,
     packages_dir: Optional[Path] = None,
@@ -246,45 +255,54 @@ def build_deployment_command(  # pylint: disable=too-many-arguments, too-many-lo
         message = f"No such file or directory: {keys_file}. Please provide valid path for keys file."
         raise click.ClickException(message)
 
-    build_dir = Path(
-        output_dir or DEFAULT_BUILD_FOLDER.format(build_hash_id())
-    ).absolute()
     if dev_mode:
         packages_dir = _validate_packages_path(path=packages_dir)
         open_aea_dir = _validate_open_aea_dir(path=open_aea_dir)
 
     ctx = cast(Context, click_context.obj)
     ctx.registry_type = registry
 
-    try:
-        build_deployment(
-            keys_file=keys_file,
-            build_dir=build_dir,
-            deployment_type=deployment_type,
-            dev_mode=dev_mode,
-            number_of_agents=number_of_agents,
-            packages_dir=packages_dir,
-            open_aea_dir=open_aea_dir,
-            log_level=log_level,
-            apply_environment_variables=aev,
-            image_version=image_version,
-            use_hardhat=use_hardhat,
-            use_acn=use_acn,
-            use_tm_testnet_setup=use_tm_testnet_setup,
-            image_author=image_author,
-            resources={
-                "agent": {
-                    "limit": {"cpu": agent_cpu_limit, "memory": agent_memory_limit},
-                    "requested": {
-                        "cpu": agent_cpu_request,
-                        "memory": agent_memory_request,
-                    },
-                }
-            },
-        )
-    except (NotValidKeysFile, FileNotFoundError, FileExistsError) as e:
-        shutil.rmtree(build_dir)
-        raise click.ClickException(str(e)) from e
+    abci_build_count = len(
+        [name for name in os.listdir() if name.startswith("abci_build_")]
+    )
+
+    for service_index in range(number_of_services):
+        service_hash_id = build_hash_id()
+        build_dir = Path(
+            output_dir or DEFAULT_BUILD_FOLDER.format(service_hash_id)
+        ).absolute()
+
+        try:
+            build_deployment(
+                service_hash_id=service_hash_id,
+                service_offset=abci_build_count + service_index,
+                keys_file=keys_file,
+                build_dir=build_dir,
+                deployment_type=deployment_type,
+                dev_mode=dev_mode,
+                number_of_agents=number_of_agents,
+                packages_dir=packages_dir,
+                open_aea_dir=open_aea_dir,
+                log_level=log_level,
+                apply_environment_variables=aev,
+                image_version=image_version,
+                use_hardhat=use_hardhat,
+                use_acn=use_acn,
+                use_tm_testnet_setup=use_tm_testnet_setup,
+                image_author=image_author,
+                resources={
+                    "agent": {
+                        "limit": {"cpu": agent_cpu_limit, "memory": agent_memory_limit},
+                        "requested": {
+                            "cpu": agent_cpu_request,
+                            "memory": agent_memory_request,
+                        },
+                    }
+                },
+            )
+        except (NotValidKeysFile, FileNotFoundError, FileExistsError) as e:
+            shutil.rmtree(build_dir)
+            raise click.ClickException(str(e)) from e
 
 
 @deploy_group.command(name="run")

autonomy/cli/helpers/deployment.py

@@ -21,7 +21,6 @@
 import os
 import shutil
 import time
-from base64 import urlsafe_b64encode
 from pathlib import Path
 from typing import Dict, List, Optional, Tuple
 
@@ -42,7 +41,7 @@
 from autonomy.configurations.constants import DEFAULT_SERVICE_CONFIG_FILE
 from autonomy.configurations.loader import load_service_config
 from autonomy.constants import DEFAULT_BUILD_FOLDER
-from autonomy.deploy.base import Resources
+from autonomy.deploy.base import Resources, build_hash_id
 from autonomy.deploy.build import generate_deployment
 from autonomy.deploy.constants import (
     AGENT_KEYS_DIR,
@@ -113,11 +112,6 @@ def _load_compose_project(build_dir: Path) -> Project:
         raise
 
 
-def build_hash_id() -> str:
-    """Generate a random 4 character hash id for the deployment build directory name."""
-    return urlsafe_b64encode(bytes(os.urandom(3))).decode()
-
-
 def run_deployment(
     build_dir: Path,
     no_recreate: bool = False,
@@ -200,6 +194,8 @@ def build_deployment(  # pylint: disable=too-many-arguments, too-many-locals
     use_tm_testnet_setup: bool = False,
     image_author: Optional[str] = None,
     resources: Optional[Resources] = None,
+    service_hash_id: Optional[str] = None,
+    service_offset: int = 0,
 ) -> None:
     """Build deployment."""
 
@@ -215,7 +211,12 @@ def build_deployment(  # pylint: disable=too-many-arguments, too-many-locals
     build_dir.mkdir()
     _build_dirs(build_dir)
 
+    if service_hash_id is None:
+        service_hash_id = build_hash_id()
+
     report = generate_deployment(
+        service_hash_id=service_hash_id,
+        service_offset=service_offset,
         service_path=Path.cwd(),
         type_of_deployment=deployment_type,
         keys_file=keys_file,

autonomy/cli/replay.py

@@ -26,8 +26,8 @@
 from aea.cli.utils.click_utils import reraise_as_click_exception
 from aea.configurations.constants import PACKAGES
 
-from autonomy.cli.helpers.deployment import build_hash_id
 from autonomy.constants import DEFAULT_BUILD_FOLDER, DOCKER_COMPOSE_YAML
+from autonomy.deploy.base import build_hash_id
 from autonomy.deploy.constants import PERSISTENT_DATA_DIR, TM_STATE_DIR
 from autonomy.replay.agent import AgentRunner
 from autonomy.replay.tendermint import build_tendermint_apps

autonomy/deploy/base.py

@@ -22,6 +22,7 @@
 import json
 import logging
 import os
+from base64 import urlsafe_b64encode
 from copy import copy, deepcopy
 from pathlib import Path
 from typing import Any, Dict, List, Optional, Set, Tuple, Union, cast
@@ -144,6 +145,11 @@ class Resources(TypedDict):
 )
 
 
+def build_hash_id() -> str:
+    """Generate a random 4 character hash id for the deployment build directory name."""
+    return urlsafe_b64encode(bytes(os.urandom(3))).decode()
+
+
 class ServiceBuilder:  # pylint: disable=too-many-instance-attributes
     """Class to assist with generating deployments."""
 
@@ -156,6 +162,8 @@ def __init__(  # pylint: disable=too-many-arguments
         keys: Optional[List[Union[List[Dict[str, str]], Dict[str, str]]]] = None,
         agent_instances: Optional[List[str]] = None,
         apply_environment_variables: bool = False,
+        service_hash_id: Optional[str] = None,
+        service_offset: int = 0,
     ) -> None:
         """Initialize the Base Deployment."""
         if apply_environment_variables:
@@ -167,8 +175,14 @@ def __init__(  # pylint: disable=too-many-arguments
             )
 
         self.service = service
+        self.service_hash_id = (
+            build_hash_id() if service_hash_id is None else service_hash_id
+        )
+        self.service_offset = service_offset
 
-        self._service_name_clean = self.service.name.replace("_", "")
+        self._service_name_clean = (
+            self.service.name.replace("_", "") + self.service_hash_id
+        )
         self._keys = keys or []
         self._agent_instances = agent_instances
         self._all_participants = self.try_get_all_participants()
@@ -254,6 +268,8 @@ def from_dir(  # pylint: disable=too-many-arguments
         agent_instances: Optional[List[str]] = None,
         apply_environment_variables: bool = False,
         dev_mode: bool = False,
+        service_hash_id: Optional[str] = None,
+        service_offset: int = 0,
     ) -> "ServiceBuilder":
         """Service builder from path."""
         service = load_service_config(
@@ -264,6 +280,8 @@ def from_dir(  # pylint: disable=too-many-arguments
 
         service_builder = cls(
             service=service,
+            service_hash_id=service_hash_id,
+            service_offset=service_offset,
             apply_environment_variables=apply_environment_variables,
         )
 

autonomy/deploy/build.py

@@ -33,6 +33,8 @@
 
 
 def generate_deployment(  # pylint: disable=too-many-arguments, too-many-locals
+    service_hash_id: str,
+    service_offset: int,
     type_of_deployment: str,
     keys_file: Path,
     service_path: Path,
@@ -66,6 +68,8 @@ def generate_deployment(  # pylint: disable=too-many-arguments, too-many-locals
         agent_instances=agent_instances,
         apply_environment_variables=apply_environment_variables,
         dev_mode=dev_mode,
+        service_hash_id=service_hash_id,
+        service_offset=service_offset,
     )
     service_builder.deplopyment_type = type_of_deployment
     service_builder.log_level = log_level

autonomy/deploy/generators/docker_compose/base.py

@@ -21,8 +21,9 @@
 import ipaddress
 import os
 from pathlib import Path
-from typing import Dict, List, Optional, cast
+from typing import Dict, List, Optional, Set, cast
 
+import yaml
 from aea.configurations.constants import (
     DEFAULT_LEDGER,
     LEDGER,
@@ -33,6 +34,7 @@
 from docker.constants import DEFAULT_NPIPE, IS_WINDOWS_PLATFORM
 from docker.errors import DockerException
 
+from autonomy.configurations.constants import DEFAULT_SERVICE_CONFIG_FILE
 from autonomy.constants import (
     ACN_IMAGE_NAME,
     ACN_IMAGE_VERSION,
@@ -192,10 +194,12 @@ def __init__(
         self,
         name: str,
         base: ipaddress.IPv4Network = BASE_SUBNET,
+        used_subnets: Optional[Set[str]] = None,
     ) -> None:
         """Initialize."""
         self.name = name
         self.base = base
+        self.used_subnets = set() if used_subnets is None else used_subnets
         self.subnet = self.build()
 
         self._address_offeset = NETWORK_ADDRESS_OFFSET
@@ -214,7 +218,7 @@ def build(
     ) -> ipaddress.IPv4Network:
         """Initialize network params."""
         docker = get_docker_client()
-        used_subnets = set()
+        used_subnets = self.used_subnets
         for network in docker.networks.list():
             # Network already exists
             if f"abci_build_{self.name}" == network.attrs["Name"]:
@@ -245,6 +249,55 @@ class DockerComposeGenerator(BaseDeploymentGenerator):
     output_name: str = DOCKER_COMPOSE_YAML
     deployment_type: str = "docker-compose"
 
+    def _find_occupied_networks(self, network_name: str) -> Set[str]:
+        """Find occupied networks on the host system."""
+        used_ports: Dict[str, str] = {}  # find occupied ports
+        for config_name in ("agent", "tendermint"):
+            for port_mapping in (
+                self.service_builder.service.deployment_config.get(config_name, {})
+                .get("ports", {})
+                .values()
+            ):
+                for host_port, _ in port_mapping.items():
+                    if host_port in used_ports:
+                        print(
+                            f"WARNING: Port {host_port} is already used by {used_ports[host_port]}."
+                        )
+                    else:
+                        used_ports[str(host_port)] = DEFAULT_SERVICE_CONFIG_FILE
+
+        used_networks = set()  # find occupied networks
+        for build in Path.cwd().glob("abci_build_*"):
+            if not (build / DOCKER_COMPOSE_YAML).exists():
+                continue
+
+            compose_config = yaml.safe_load((build / DOCKER_COMPOSE_YAML).read_text())
+            for subnet in (
+                compose_config.get("networks", {})
+                .get(network_name, {})
+                .get("ipam", {})
+                .get("config", [])
+            ):
+                used_networks.add(subnet["subnet"])
+
+            services = compose_config.get("services", {})
+            for name, service in services.items():
+                for port_mapping_str in service.get("ports", []):
+                    port_mapping = port_mapping_str.split(":")
+                    if len(port_mapping) != 2:
+                        continue
+
+                    host_port = port_mapping[0]
+                    if host_port in used_ports:
+                        print(
+                            f"WARNING: Port {host_port} is already used by {used_ports[host_port]}."
+                            f" Please adjust the port in {build / DOCKER_COMPOSE_YAML} manually.",
+                        )
+                    else:
+                        used_ports[host_port] = name
+
+        return used_networks
+
     def generate_config_tendermint(self) -> "DockerComposeGenerator":
         """Generate the command to configure tendermint testnet."""
         if not self.use_tm_testnet_setup:
@@ -280,9 +333,9 @@ def generate(
         use_acn: bool = False,
     ) -> "DockerComposeGenerator":
         """Generate the new configuration."""
-
         network_name = f"service_{self.service_builder.service.name}_localnet"
-        network = Network(name=network_name)
+        used_networks = self._find_occupied_networks(network_name)
+        network = Network(name=network_name, used_subnets=used_networks)
         image_version = image_version or self.service_builder.service.agent.hash
         if self.dev_mode:
             runtime_image = DEVELOPMENT_IMAGE