diff --git a/src/optimade_launch/README.md b/src/optimade_launch/README.md index f5e3ce7..6fd7497 100644 --- a/src/optimade_launch/README.md +++ b/src/optimade_launch/README.md @@ -14,29 +14,39 @@ To use OPTIMADE launch you will have to _Or directly with pip (`pip install optimade-launch`)._ -3. creating a profile and attach a database from inject data from JSONL file +3. creating a profile with a config yaml file + + ```yml + name: + socket: + - /host/path.sock + - /var/lib/optimade-sockets/.sock + jsonl: + - /var/lib/optimade-archive/.jsonl + mongo_uri: mongodb://localhost:27017 + db_name: + optimade_base_url: http://localhost + optimade_index_base_url: http://localhost + optimade_provider: + prefix: "myorg" + name: "My Organization" + description: "My Organization's OPTIMADE provider" + homepage: "https://myorg.org" + ``` ```console - optimade-launch profile create --profile-name test --jsonl /path/to/your/jsonl/file + optimade-launch profile create --config /path/to/your/yml/file ``` 4. Start OPTIMADE server of testing data with ```console - optimade-launch start + optimade-launch server start -p ``` 5. Follow the instructions on screen to open OPTIMADE API in the browser. See `optimade-launch --help` for detailed help. -### Instance Management - -You can inspect the status of all configured AiiDAlab profiles with: - -```console -optimade-launch status -``` - ### Profile Management The tool allows to manage multiple profiles, e.g., with different home directories or ports. @@ -53,7 +63,7 @@ Can used to clean up the database. See `optimade-launch container --help` for more information. -Can be used to check the status of the container, or to stop and remove the container. +Can be used to start the created container, or to stop and remove the container. ### Server Management diff --git a/src/optimade_launch/optimade_launch/__main__.py b/src/optimade_launch/optimade_launch/__main__.py index d9ef479..ebd6ecb 100644 --- a/src/optimade_launch/optimade_launch/__main__.py +++ b/src/optimade_launch/optimade_launch/__main__.py @@ -10,7 +10,7 @@ from .util import spinner from .instance import OptimadeInstance, _BUILD_TAG from .application_state import ApplicationState -from .profile import DEFAULT_PORT, Profile +from .profile import Profile from .version import __version__ LOGGING_LEVELS = { @@ -112,8 +112,7 @@ async def _async_start( ) except docker.errors.APIError as error: - # TODO LOGGING - raise click.ClickException("Startup failed due to an API error.") from error + raise except Exception as error: raise click.ClickException(f"Unknown error: {error}.") from error @@ -349,7 +348,7 @@ def edit_profile(app_state, profile): "--jsonl", type=click.Path(exists=True), multiple=True, - help="Path to a JSON Lines file as the source of database.", + help="Path to a OPTIMADE JSON Lines file as the source of database.", ) @click.option( "--db-name", @@ -360,7 +359,7 @@ def edit_profile(app_state, profile): "--config", type=click.Path(exists=True), required=False, - help="Path to a YAML file containing the configuration.", + help="Path to a YAML configuration to create a profile from.", ) @pass_app_state @click.pass_context @@ -368,12 +367,18 @@ def create_profile(ctx, app_state, port: int | None, mongo_uri: str, jsonl: list """Add a new Optimade profile to the configuration.""" import json + # XXX: read config if exist and use it as base, override with cli args if provided if config: import yaml with open(config) as f: params = yaml.safe_load(f) - profile = params["name"] + if "name" in params: + profile = params["name"] + elif profile is None: + raise click.ClickException("No profile name provided.") + else: + params["name"] = profile else: params = { "name": profile, diff --git a/src/optimade_launch/optimade_launch/application_state.py b/src/optimade_launch/optimade_launch/application_state.py index 620a1c5..31a578f 100644 --- a/src/optimade_launch/optimade_launch/application_state.py +++ b/src/optimade_launch/optimade_launch/application_state.py @@ -6,18 +6,15 @@ import click import docker -from packaging.version import parse from .config import Config -from .core import APPLICATION_ID -from .instance import OptimadeInstance -from .profile import Profile +from optimade_launch.core import CONFIG_FOLDER from .util import get_docker_client from .version import __version__ def _application_config_path(): - return Path(click.get_app_dir(APPLICATION_ID)) / "config.toml" + return CONFIG_FOLDER / "config.toml" def _load_config(): @@ -35,33 +32,3 @@ class ApplicationState: def save_config(self): self.config.save(self.config_path) - - # def _apply_migration_null(self): - # # Since there is no config file on disk, we can assume that if at all, - # # there is only the default profile present. - # assert len(self.config.profiles) == 1 - # assert self.config.profiles[0].name == "default" - - # default_profile = self.config.profiles[0] - # instance = OptimadeInstance(client=self.docker_client, profile=default_profile) - - # if instance.container: - # # There is already a container present, use previously used profile. - # self.config.profiles[0] = Profile.from_container(instance.container) - - # def apply_migrations(self): - # config_changed = False - - # # No config file saved to disk. - # if not self.config_path.is_file(): - # self._apply_migration_null() - # config_changed = True - - # # No version string stored in config. - # if self.config.version != str(parse(__version__)): - # self.config.version = str(parse(__version__)) - # config_changed = True - - # # Write any changes back to disk. - # if config_changed: - # self.save_config() diff --git a/src/optimade_launch/optimade_launch/core.py b/src/optimade_launch/optimade_launch/core.py index 121c695..8644731 100644 --- a/src/optimade_launch/optimade_launch/core.py +++ b/src/optimade_launch/optimade_launch/core.py @@ -1,7 +1,14 @@ # __future__ import needed for classmethod factory functions; should be dropped # with py 3.10. +import os import logging +import click +from pathlib import Path APPLICATION_ID = "org.optimade.optimade_launch" - LOGGER = logging.getLogger(APPLICATION_ID.split(".")[-1]) + +if os.environ.get("OPTIMADE_LAUNCH_CONFIG_FOLDER"): + CONFIG_FOLDER = Path(os.environ["OPTIMADE_LAUNCH_CONFIG_FOLDER"]) +else: + CONFIG_FOLDER = Path(click.get_app_dir(APPLICATION_ID)) diff --git a/src/optimade_launch/optimade_launch/dockerfiles/Dockerfile b/src/optimade_launch/optimade_launch/dockerfiles/Dockerfile index fca2f4a..f71f47c 100644 --- a/src/optimade_launch/optimade_launch/dockerfiles/Dockerfile +++ b/src/optimade_launch/optimade_launch/dockerfiles/Dockerfile @@ -1,4 +1,5 @@ -FROM ghcr.io/materials-consortia/optimade:0.24.0 +ARG BASE_IMAGE +FROM ${BASE_IMAGE} # copy repo contents and install deps COPY requirements.txt ./ @@ -8,4 +9,8 @@ ENV UNIX_SOCK "/tmp/gunicorn.sock" COPY run.sh /app/run.sh -ENV OPTIMADE_CONFIG_FILE "" \ No newline at end of file +## Set the default config file path +## Then copy the config from host to container +#ENV OPTIMADE_CONFIG_FILE "/config/config.yml" +# +#COPY optimade-config.yml ${OPTIMADE_CONFIG_FILE} \ No newline at end of file diff --git a/src/optimade_launch/optimade_launch/instance.py b/src/optimade_launch/optimade_launch/instance.py index d1df06b..88156a9 100644 --- a/src/optimade_launch/optimade_launch/instance.py +++ b/src/optimade_launch/optimade_launch/instance.py @@ -110,9 +110,13 @@ def build(self, tag: None | str = None) -> docker.models.images.Image: try: LOGGER.info(f"Building image from Dockerfile: {str(_DOCKERFILE_PATH)}") tag = tag or _BUILD_TAG + buildargs = { + "BASE_IMAGE": self.profile.image, + } image, logs = self.client.images.build( path=str(_DOCKERFILE_PATH), tag=tag, + buildargs=buildargs, rm=True, ) LOGGER.info(f"Built image: {image}") @@ -187,11 +191,15 @@ def create(self, data: bool = False) -> Container: sock_filename = os.path.basename(self.profile.unix_sock) params_container["volumes"] = {host_sock_folder: {"bind": '/tmp', "mode": "rw"}} environment["UNIX_SOCK"] = f"/tmp/{sock_filename}" + + # optimade config file + environment["OPTIMADE_CONFIG_FILE"] = self.profile.optimade_config_file if sys.platform == "linux" and "host.docker.internal" in self.profile.mongo_uri: params_container["extra_hosts"] = {"host.docker.internal": "host-gateway"} params_container["environment"] = environment + self._container = self.client.containers.create( **params_container, ) @@ -205,7 +213,7 @@ def recreate(self) -> None: self.create() def start(self) -> None: - # TODO: check mongodb can be connected to + # XXX: check mongodb can be connected to LOGGER.info(f"Starting container '{self.profile.container_name()}'...") (self.container or self.create(data=True)).start() assert self.container is not None diff --git a/src/optimade_launch/optimade_launch/profile.py b/src/optimade_launch/optimade_launch/profile.py index 64027a3..38a72b1 100644 --- a/src/optimade_launch/optimade_launch/profile.py +++ b/src/optimade_launch/optimade_launch/profile.py @@ -10,7 +10,7 @@ CONTAINER_PREFIX = "optimade" DEFAULT_PORT = 8081 -DEFAULT_IMAGE = "ghcr.io/materials-consortia/optimade:0.24.0" +DEFAULT_IMAGE = "ghcr.io/materials-consortia/optimade:0.25.2" DEFAULT_MONGO_URI = "mongodb://127.0.0.1:27017" DEFAULT_BASE_URL = "http://localhost" DEFAULT_INDEX_BASE_URL = "http://localhost" @@ -35,14 +35,16 @@ def _get_configured_host_port(container: Container) -> int | None: @dataclass class Profile: name: str = DEFAULT_NAME + image: str = DEFAULT_IMAGE jsonl_paths: list[str] = field(default_factory=lambda: []) mongo_uri: str = DEFAULT_MONGO_URI db_name: str = "optimade" port: int | None = None unix_sock: str | None = None - optimade_base_url: str | None = DEFAULT_BASE_URL - optimade_index_base_url: str | None = DEFAULT_INDEX_BASE_URL - optimade_provider: str | None = DEFAULT_PROVIDER + optimade_config_file: str | None = None + optimade_base_url: str | None = None + optimade_index_base_url: str | None = None + optimade_provider: str | None = None optimade_validate_api_response: bool = False def __post_init__(self): @@ -61,7 +63,7 @@ def environment(self) -> dict: self.mongo_uri = self.mongo_uri.replace("127.0.0.1", "host.docker.internal") return { - "OPTIMADE_CONFIG_FILE": None, + "optimade_config_file": self.optimade_config_file, "optimade_insert_test_data": False, "optimade_database_backend": "mongodb", "optimade_mongo_uri": self.mongo_uri, @@ -77,7 +79,7 @@ def environment(self) -> dict: def dumps(self) -> str: """Dump the profile to a TOML string.""" - return toml.dumps({k: v for k, v in asdict(self).items() if k != "name"}) + return toml.dumps({k: v for k, v in asdict(self).items() if k != "name" and v is not None}) @classmethod def loads(cls, name: str, s: str) -> Profile: diff --git a/src/optimade_launch/pyproject.toml b/src/optimade_launch/pyproject.toml index c3ca78e..df314d5 100644 --- a/src/optimade_launch/pyproject.toml +++ b/src/optimade_launch/pyproject.toml @@ -22,7 +22,7 @@ dependencies = [ ] [project.optional-dependencies] -tests = [ +dev = [ "pytest~=7.0.1", "pytest-asyncio==0.20.3", "pytest-mock-resources~=2.7.0", diff --git a/src/optimade_launch/tests/_static/config.yaml b/src/optimade_launch/tests/_static/config.yaml index 44fce93..a00d0a8 100644 --- a/src/optimade_launch/tests/_static/config.yaml +++ b/src/optimade_launch/tests/_static/config.yaml @@ -1,10 +1,11 @@ --- name: te-st jsonl_paths: - - /var/lib/optimade-archive/te-st.jsonl + - /tmp/optimade.jsonl mongo_uri: mongodb://localhost:27017 db_name: te-st -unix_sock: /var/lib/optimade-sockets/te-st.sock +unix_sock: /tmp/te-st.sock +optimade_config_file: /tmp/optimade_config.json optimade_base_url: http://localhost optimade_index_base_url: http://localhost optimade_provider: diff --git a/src/optimade_launch/tests/_static/optimade_config.json b/src/optimade_launch/tests/_static/optimade_config.json new file mode 100644 index 0000000..1c47ca9 --- /dev/null +++ b/src/optimade_launch/tests/_static/optimade_config.json @@ -0,0 +1,38 @@ +{ + "debug": false, + "default_db": "test_server", + "base_url": "http://localhost:5000", + "implementation": { + "name": "Example implementation", + "source_url": "https://github.com/Materials-Consortia/optimade-python-tools", + "issue_tracker": "https://github.com/Materials-Consortia/optimade-python-tools/issues", + "maintainer": {"email": "dev@optimade.org"} + }, + "provider": { + "name": "Example provider", + "description": "Provider used for examples, not to be assigned to a real database", + "prefix": "exmpl", + "homepage": "https://example.com" + }, + "index_base_url": "http://localhost:5001", + "provider_fields": { + "structures": [ + "band_gap", + {"name": "chemsys", "type": "string", "description": "A string representing the chemical system in an ordered fashion"} + ] + }, + "aliases": { + "structures": { + "id": "task_id", + "immutable_id": "_id", + "chemical_formula_descriptive": "pretty_formula", + "chemical_formula_reduced": "pretty_formula", + "chemical_formula_anonymous": "formula_anonymous" + } + }, + "length_aliases": { + "structures": { + "chemsys": "nelements" + } + } +} \ No newline at end of file diff --git a/src/optimade_launch/tests/conftest.py b/src/optimade_launch/tests/conftest.py index 18cc43b..3b4d489 100644 --- a/src/optimade_launch/tests/conftest.py +++ b/src/optimade_launch/tests/conftest.py @@ -96,6 +96,8 @@ async def started_instance(docker_client, monkeypatch, mongo, static_dir): if host in ("localhost", "127.0.0.1"): instance._container.update({"network_mode": "host"}) + + print(instance.profile.environment()) instance.create(data=True) assert instance.container is not None diff --git a/src/optimade_launch/tests/test_cli.py b/src/optimade_launch/tests/test_cli.py index 25ca82a..d3b50c5 100644 --- a/src/optimade_launch/tests/test_cli.py +++ b/src/optimade_launch/tests/test_cli.py @@ -1,8 +1,3 @@ -import logging -from dataclasses import replace - -import docker -import pytest from click.testing import CliRunner, Result import optimade_launch.__main__ as cli @@ -13,9 +8,11 @@ def test_version_displays_library_version(): """Test that the CLI displays the library version. """ runner: CliRunner = CliRunner() + result: Result = runner.invoke(cli.cli, ["version"]) assert __version__ in result.output.strip(), "Version number should match library version." assert "Optimade Launch" in result.output.strip() + def test_list_profiles(): runner: CliRunner = CliRunner() diff --git a/src/optimade_launch/tests/test_config.py b/src/optimade_launch/tests/test_config.py index ded5f57..1d88e2d 100644 --- a/src/optimade_launch/tests/test_config.py +++ b/src/optimade_launch/tests/test_config.py @@ -11,7 +11,7 @@ [profiles.default] port = 8081 - image = "ghcr.io/materials-consortia/optimade:0.24.0" + image = "ghcr.io/materials-consortia/optimade:latest" """ } diff --git a/src/optimade_launch/tests/test_instance.py b/src/optimade_launch/tests/test_instance.py index 8cf811e..3219296 100644 --- a/src/optimade_launch/tests/test_instance.py +++ b/src/optimade_launch/tests/test_instance.py @@ -1,13 +1,8 @@ import pytest -from optimade_launch.profile import Profile import docker from pathlib import PurePosixPath -from dataclasses import replace -from optimade_launch.instance import RequiresContainerInstance, OptimadeInstance -import re - -from pymongo import MongoClient +from optimade_launch.instance import RequiresContainerInstance @pytest.mark.asyncio async def test_instance_init(instance): @@ -66,35 +61,4 @@ async def test_instance_create_and_check_sock(instance): mount = get_docker_mount(instance.container, PurePosixPath("/tmp")) assert mount["Type"] == "bind" assert mount["Source"] == "/tmp/optimade-sock" - -# start a instance and test real actions -@pytest.mark.usefixtures("started_instance") -class TestsAgainstStartedInstance: - - @pytest.mark.asyncio - async def test_instance_status(self, started_instance): - assert ( - await started_instance.status() - is started_instance.OptimadeInstanceStatus.UP - ) - - def test_instance_url(self, started_instance): - assert re.match( - r"http:\/\/localhost:\d+\/", started_instance.url() - ) - - def test_instance_host_ports(self, started_instance): - assert len(started_instance.host_ports()) > 0 - - @pytest.mark.asyncio - async def test_instance_query(self, started_instance): - """make a query to the instance""" - import requests - assert ( - await started_instance.status() - is started_instance.OptimadeInstanceStatus.UP - ) - - response = requests.get(started_instance.url() + "v1/structures") - assert response.status_code == 200 - assert response.json()["meta"]["data_available"] == 3 + \ No newline at end of file diff --git a/src/optimade_launch/tests/test_started_instance.py b/src/optimade_launch/tests/test_started_instance.py new file mode 100644 index 0000000..088b7b0 --- /dev/null +++ b/src/optimade_launch/tests/test_started_instance.py @@ -0,0 +1,34 @@ +import pytest +import re + +# start a instance and test real actions +@pytest.mark.usefixtures("started_instance") +class TestsAgainstStartedInstance: + + @pytest.mark.asyncio + async def test_instance_status(self, started_instance): + assert ( + await started_instance.status() + is started_instance.OptimadeInstanceStatus.UP + ) + + def test_instance_url(self, started_instance): + assert re.match( + r"http:\/\/localhost:\d+\/", started_instance.url() + ) + + def test_instance_host_ports(self, started_instance): + assert len(started_instance.host_ports()) > 0 + + @pytest.mark.asyncio + async def test_instance_query(self, started_instance): + """make a query to the instance""" + import requests + assert ( + await started_instance.status() + is started_instance.OptimadeInstanceStatus.UP + ) + + response = requests.get(started_instance.url() + "v1/structures") + assert response.status_code == 200 + assert response.json()["meta"]["data_available"] == 3 \ No newline at end of file