Merge pull request #2085 from valory-xyz/bump/open-aea-1-41-0

Bump [email protected]

Pipfile

@@ -7,10 +7,10 @@ name = "pypi"
 aiohttp = "<3.8,>=3.7.4"
 docker = "==6.1.2"
 Flask = "==2.0.2"
-open-aea = {version = "==1.40.0", extras = ["all"]}
-open-aea-ledger-ethereum = "==1.40.0"
-open-aea-ledger-ethereum-hwi = "==1.40.0"
-open-aea-cli-ipfs = "==1.40.0"
+open-aea = {version = "==1.41.0", extras = ["all"]}
+open-aea-ledger-ethereum = "==1.41.0"
+open-aea-ledger-ethereum-hwi = "==1.41.0"
+open-aea-cli-ipfs = "==1.41.0"
 ipfshttpclient = "==0.8.0a2"
 Werkzeug= "==2.0.3"
 watchdog = ">=2.1.6"
@@ -32,7 +32,7 @@ typing_extensions = ">=3.10.0.2"
 hexbytes = "*"
 packaging = "*"
 pytest-asyncio = "*"
-open-aea-ledger-cosmos = "==1.40.0"
+open-aea-ledger-cosmos = "==1.41.0"
 # we pin this as the range specified in open-aea-ledger-cosmos is wide
 open-aea-cosmpy = "==0.6.6"
 grpcio = "==1.53.0"

autonomy/cli/build_images.py

@@ -19,18 +19,20 @@
 
 """Build images."""
 
-
 from pathlib import Path
-from typing import Optional
+from typing import Optional, Tuple
 
 import click
-from aea.cli.utils.click_utils import PublicIdParameter, reraise_as_click_exception
-from aea.configurations.data_types import PublicId
+from aea.cli.utils.click_utils import (
+    PublicIdParameter,
+    PyPiDependency,
+    reraise_as_click_exception,
+)
+from aea.configurations.data_types import Dependency, PublicId
 
+from autonomy.cli.helpers.image import build_image as _build_image
 from autonomy.cli.utils.click_utils import image_author_option
-from autonomy.configurations.loader import load_service_config
 from autonomy.deploy.image import ImageBuildFailed
-from autonomy.deploy.image import build_image as _build_image
 
 
 @click.command(name="build-image")
@@ -44,28 +46,36 @@
     type=click.Path(dir_okay=True),
     help="Path to service dir.",
 )
+@click.option(
+    "-e",
+    "--extra-dependency",
+    "extra_dependencies",
+    type=PyPiDependency(),
+    help="Provide extra dependency.",
+    multiple=True,
+)
 @click.option("--version", type=str, help="Specify tag version for the image.")
 @click.option("--dev", is_flag=True, help="Build development image.", default=False)
 @click.option("--pull", is_flag=True, help="Pull latest dependencies.", default=False)
 @image_author_option
-def build_image(
+def build_image(  # pylint: disable=too-many-arguments
     agent: Optional[PublicId],
     service_dir: Optional[Path],
+    extra_dependencies: Tuple[Dependency, ...],
     pull: bool = False,
     dev: bool = False,
     version: Optional[str] = None,
     image_author: Optional[str] = None,
 ) -> None:
     """Build runtime images for autonomous agents."""
 
-    with reraise_as_click_exception(FileNotFoundError):
-        if agent is None:
-            service_dir = Path(service_dir or Path.cwd()).absolute()
-            service = load_service_config(service_dir)
-            agent = service.agent
-
-    with reraise_as_click_exception(ImageBuildFailed):
-        click.echo(f"Building image with agent: {agent}\n")
+    with reraise_as_click_exception(ImageBuildFailed, FileNotFoundError):
         _build_image(
-            agent=agent, pull=pull, dev=dev, version=version, image_author=image_author
+            agent=agent,
+            service_dir=service_dir,
+            extra_dependencies=extra_dependencies,
+            pull=pull,
+            dev=dev,
+            version=version,
+            image_author=image_author,
         )

autonomy/cli/helpers/image.py

@@ -0,0 +1,57 @@
+# -*- coding: utf-8 -*-
+# ------------------------------------------------------------------------------
+#
+#   Copyright 2023 Valory AG
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+# ------------------------------------------------------------------------------
+
+"""Image helpers."""
+
+from pathlib import Path
+from typing import Optional, Tuple
+
+import click
+from aea.configurations.data_types import Dependency, PublicId
+
+from autonomy.configurations.loader import load_service_config
+from autonomy.deploy.image import build_image as _build_image
+
+
+def build_image(  # pylint: disable=too-many-arguments
+    agent: Optional[PublicId],
+    service_dir: Optional[Path],
+    pull: bool = False,
+    dev: bool = False,
+    version: Optional[str] = None,
+    image_author: Optional[str] = None,
+    extra_dependencies: Optional[Tuple[Dependency, ...]] = None,
+) -> None:
+    """Build agent/service image."""
+    extra_dependencies = extra_dependencies or ()
+    if agent is None:
+        service_dir = Path(service_dir or Path.cwd()).absolute()
+        service = load_service_config(service_dir)
+        agent = service.agent
+        extra_dependencies = (*extra_dependencies, *service.dependencies.values())
+
+    click.echo(f"Building image with agent: {agent}\n")
+    _build_image(
+        agent=agent,
+        pull=pull,
+        dev=dev,
+        version=version,
+        image_author=image_author,
+        extra_dependencies=extra_dependencies,
+    )

autonomy/configurations/base.py

@@ -35,7 +35,12 @@
     PACKAGE_TYPE_TO_CONFIG_CLASS as _PACKAGE_TYPE_TO_CONFIG_CLASS,
 )
 from aea.configurations.base import PackageConfiguration, ProtocolConfig, SkillConfig
-from aea.configurations.data_types import PackageType, PublicId
+from aea.configurations.data_types import (
+    Dependencies,
+    Dependency,
+    PackageType,
+    PublicId,
+)
 from aea.exceptions import AEAValidationError
 from aea.helpers.base import SimpleIdOrStr
 from aea.helpers.env_vars import apply_env_variables, generate_env_vars_recursively
@@ -55,6 +60,13 @@
 }
 
 
+def load_dependencies(dependencies: Dict) -> Dependencies:
+    """Load dependencies."""
+    return {
+        name: Dependency.from_json({name: spec}) for name, spec in dependencies.items()
+    }
+
+
 class Service(PackageConfiguration):  # pylint: disable=too-many-instance-attributes
     """Service package configuration."""
 
@@ -75,6 +87,7 @@ class Service(PackageConfiguration):  # pylint: disable=too-many-instance-attrib
         "agent",
         "number_of_agents",
         "description",
+        "dependencies",
         "deployment_config",
         "_aea_version",
         "_aea_version_specifiers",
@@ -97,6 +110,7 @@ def __init__(  # pylint: disable=too-many-arguments
         build_entrypoint: Optional[str] = None,
         overrides: Optional[List] = None,
         deployment: Optional[Dict] = None,
+        dependencies: Optional[Dependencies] = None,
     ) -> None:
         """Initialise object."""
 
@@ -115,6 +129,7 @@ def __init__(  # pylint: disable=too-many-arguments
         self.description = description
         self.number_of_agents = number_of_agents
         self.deployment_config = deployment or {}
+        self.dependencies = dependencies or {}
 
         self._overrides = [] if overrides is None else overrides
 

autonomy/configurations/loader.py

@@ -34,7 +34,7 @@
 from aea.helpers.io import open_file
 from aea.helpers.yaml_utils import yaml_load_all
 
-from autonomy.configurations.base import Service
+from autonomy.configurations.base import Service, load_dependencies
 
 
 COMPONENT_CONFIGS: Dict = {
@@ -74,10 +74,24 @@ def load_service_config(
     service_config = apply_env_variables(
         service_config, env_variables=os.environ.copy()
     )
+
+    if "dependencies" in service_config:
+        dependencies = load_dependencies(
+            dependencies=service_config.pop("dependencies")
+        )
+    else:
+        dependencies = {}
+        warn(
+            "`dependencies` parameter not defined in the service",
+            FutureWarning,
+            stacklevel=2,
+        )
+        print("WARNING: `dependencies` parameter not defined in the service")
+
     Service.validate_config_data(service_config)
     service_config["license_"] = service_config.pop("license")
 
-    service = Service(**service_config)
+    service = Service(**service_config, dependencies=dependencies)
     service.overrides = overrides
 
     return service

autonomy/configurations/schemas/service_schema.json

@@ -1,7 +1,7 @@
 {
   "$schema": "http://json-schema.org/draft-04/schema#",
   "description": "Schema for the deployment configuration file.",
-  "additionalProperties": false,
+  "additionalProperties": true,
   "type": "object",
   "required": [
     "name",

autonomy/constants.py

@@ -58,4 +58,4 @@
 ACN_IMAGE_NAME = os.environ.get("ACN_IMAGE_NAME", "valory/open-acn-node")
 DEFAULT_DOCKER_IMAGE_AUTHOR = "valory"
 OAR_IMAGE = "{image_author}/oar-{agent}:{version}"
-ABSTRACT_ROUND_ABCI_SKILL_WITH_HASH = "valory/abstract_round_abci:0.1.0:bafybeiavfdmszwpotgdw5wd2imxcwsigygczvttxk5onswt72ipbdyjp3q"
+ABSTRACT_ROUND_ABCI_SKILL_WITH_HASH = "valory/abstract_round_abci:0.1.0:bafybeibbvbsrzavof2o4yaelujjn563pylgatbdxlsfgfxeh3gzdp7v6pi"

autonomy/data/Dockerfiles/agent/Dockerfile

@@ -1,16 +1,16 @@
 ARG AUTONOMY_IMAGE_VERSION="latest"
 ARG AUTONOMY_IMAGE_NAME="valory/open-autonomy"
-
 FROM ${AUTONOMY_IMAGE_NAME}:${AUTONOMY_IMAGE_VERSION}
 
 ARG AEA_AGENT
 ARG AUTHOR
+ARG EXTRA_DEPENDENCIES
 
 RUN aea init --reset --remote --ipfs --author ${AUTHOR}
 
 WORKDIR /root
 
-RUN AEA_AGENT=${AEA_AGENT} bash /root/scripts/install.sh
+RUN AEA_AGENT=${AEA_AGENT} EXTRA_DEPENDENCIES=${EXTRA_DEPENDENCIES} bash /root/scripts/install.sh
 
 CMD ["/root/scripts/start.sh"]
 

autonomy/deploy/base.py

@@ -54,6 +54,7 @@
 ENV_VAR_AEA_AGENT = "AEA_AGENT"
 ENV_VAR_LOG_LEVEL = "LOG_LEVEL"
 ENV_VAR_AEA_PASSWORD = "AEA_PASSWORD"  # nosec
+ENV_VAR_DEPENDENCIES = "DEPENDENCIES"  # nosec
 
 PARAM_ARGS_PATH = ("models", "params", "args")
 SETUP_PARAM_PATH = (*PARAM_ARGS_PATH, "setup")

autonomy/deploy/image.py

@@ -21,10 +21,10 @@
 
 
 import json
-from typing import Dict, Optional
+from typing import Dict, Optional, Tuple
 
 from aea.cli.utils.config import get_default_author_from_cli_config
-from aea.configurations.utils import PublicId
+from aea.configurations.data_types import Dependency, PublicId
 from docker import from_env
 
 from autonomy.constants import (
@@ -37,6 +37,16 @@
 from autonomy.deploy.constants import DOCKERFILES
 
 
+def generate_dependency_flag_var(dependencies: Tuple[Dependency, ...]) -> str:
+    """Generate dependency flag env var"""
+    args = []
+    for dep in dependencies:
+        # Flag for `aea install` command
+        args += ["-e"]
+        args += dep.get_pip_install_args()
+    return " ".join(args)
+
+
 class ImageProfiles:  # pylint: disable=too-few-public-methods
     """Image build profiles."""
 
@@ -52,33 +62,28 @@ def build_image(
     dev: bool = False,
     version: Optional[str] = None,
     image_author: Optional[str] = None,
+    extra_dependencies: Optional[Tuple[Dependency, ...]] = None,
 ) -> None:
     """Command to build images from for skaffold deployment."""
 
     tag: str
     path: str
     buildargs: Dict[str, str]
-
     docker_client = from_env()
+    buildargs = {
+        "AUTONOMY_IMAGE_NAME": AUTONOMY_IMAGE_NAME,
+        "AUTONOMY_IMAGE_VERSION": AUTONOMY_IMAGE_VERSION,
+        "AEA_AGENT": str(agent),
+        "AUTHOR": get_default_author_from_cli_config(),
+        "EXTRA_DEPENDENCIES": generate_dependency_flag_var(extra_dependencies or ()),
+    }
 
     if dev:
         path = str(DATA_DIR / DOCKERFILES / "dev")
-        buildargs = {
-            "AUTONOMY_IMAGE_NAME": AUTONOMY_IMAGE_NAME,
-            "AUTONOMY_IMAGE_VERSION": AUTONOMY_IMAGE_VERSION,
-            "AEA_AGENT": str(agent),
-            "AUTHOR": get_default_author_from_cli_config(),
-        }
-
     else:
         image_version = version or agent.hash
         path = str(DATA_DIR / DOCKERFILES / "agent")
-        buildargs = {
-            "AUTONOMY_IMAGE_NAME": AUTONOMY_IMAGE_NAME,
-            "AUTONOMY_IMAGE_VERSION": AUTONOMY_IMAGE_VERSION,
-            "AEA_AGENT": str(agent),
-            "AUTHOR": get_default_author_from_cli_config(),
-        }
+
     tag = OAR_IMAGE.format(
         image_author=image_author or DEFAULT_DOCKER_IMAGE_AUTHOR,
         agent=agent.name,

autonomy/services/scaffold/service.yaml

@@ -7,3 +7,5 @@ license: Apache-2.0
 agent: agent
 network: hardhat
 number_of_agents: 4
+deployment: {}
+dependencies: {}

deployments/Dockerfiles/autonomy-user/requirements.txt

@@ -1,5 +1,5 @@
 open-autonomy[all]==0.13.0
-open-aea[all]==1.40.0
-open-aea-cli-ipfs==1.40.0
-open-aea-ledger-ethereum==1.40.0
+open-aea[all]==1.41.0
+open-aea-cli-ipfs==1.41.0
+open-aea-ledger-ethereum==1.41.0
 protobuf>=4.21.6,<5.0.0

deployments/Dockerfiles/autonomy/scripts/install.sh

@@ -17,7 +17,7 @@ aea build || exit 1
 echo "Successfully built the host dependencies."
 
 echo "Installing the necessary python dependencies!"
-aea install || exit 1
+aea install $EXTRA_DEPENDENCIES || exit 1
 echo "Successfully Installed the python dependecies."
 
 echo "Done."