optimade-launch accept config file to start the server (#29)

- Fix failed tests (the started container tests are still failing)
- Update the README to conform with the current CLI interface
- Allow to pass the optimade config file to set the profile and start the server with it. This allows provider information to be much clearer.

src/optimade_launch/README.md

@@ -14,29 +14,39 @@ To use OPTIMADE launch you will have to
 
    _Or directly with pip (`pip install optimade-launch`)._
 
-3. creating a profile and attach a database from inject data from JSONL file
+3. creating a profile with a config yaml file
+
+   ```yml
+   name: <parsed_doi>
+   socket:
+      - /host/path.sock
+      - /var/lib/optimade-sockets/<parsed_doi>.sock
+   jsonl:
+      - /var/lib/optimade-archive/<parsed_doi>.jsonl
+   mongo_uri: mongodb://localhost:27017
+   db_name: <parsed_doi>
+   optimade_base_url: http://localhost
+   optimade_index_base_url: http://localhost
+   optimade_provider:
+      prefix: "myorg"
+      name: "My Organization"
+      description: "My Organization's OPTIMADE provider"
+      homepage: "https://myorg.org"
+   ```
 
    ```console
-   optimade-launch profile create --profile-name test --jsonl /path/to/your/jsonl/file
+   optimade-launch profile create --config /path/to/your/yml/file
    ```
 
 4. Start OPTIMADE server of testing data with
 
     ```console
-    optimade-launch start
+    optimade-launch server start -p <profile_name>
     ```
 5. Follow the instructions on screen to open OPTIMADE API in the browser.
 
 See `optimade-launch --help` for detailed help.
 
-### Instance Management
-
-You can inspect the status of all configured AiiDAlab profiles with:
-
-```console
-optimade-launch status
-```
-
 ### Profile Management
 
 The tool allows to manage multiple profiles, e.g., with different home directories or ports.
@@ -53,7 +63,7 @@ Can used to clean up the database.
 
 See `optimade-launch container --help` for more information.
 
-Can be used to check the status of the container, or to stop and remove the container.
+Can be used to start the created container, or to stop and remove the container.
 
 ### Server Management
 

src/optimade_launch/optimade_launch/__main__.py

@@ -10,7 +10,7 @@
 from .util import spinner
 from .instance import OptimadeInstance, _BUILD_TAG
 from .application_state import ApplicationState
-from .profile import DEFAULT_PORT, Profile
+from .profile import Profile
 from .version import __version__
 
 LOGGING_LEVELS = {
@@ -112,8 +112,7 @@ async def _async_start(
             )
 
     except docker.errors.APIError as error:
-        # TODO LOGGING
-        raise click.ClickException("Startup failed due to an API error.") from error
+        raise
 
     except Exception as error:
         raise click.ClickException(f"Unknown error: {error}.") from error
@@ -349,7 +348,7 @@ def edit_profile(app_state, profile):
     "--jsonl", 
     type=click.Path(exists=True), 
     multiple=True,
-    help="Path to a JSON Lines file as the source of database.",
+    help="Path to a OPTIMADE JSON Lines file as the source of database.",
 )
 @click.option(
     "--db-name",
@@ -360,20 +359,26 @@ def edit_profile(app_state, profile):
     "--config",
     type=click.Path(exists=True),
     required=False,
-    help="Path to a YAML file containing the configuration.",
+    help="Path to a YAML configuration to create a profile from.",
 )
 @pass_app_state
 @click.pass_context
 def create_profile(ctx, app_state, port: int | None, mongo_uri: str, jsonl: list, db_name, config, profile: str | None = None):
     """Add a new Optimade profile to the configuration."""
     import json
 
+    # XXX: read config if exist and use it as base, override with cli args if provided
     if config:
         import yaml
 
         with open(config) as f:
             params = yaml.safe_load(f)
-            profile = params["name"]
+            if "name" in params:
+                profile = params["name"]
+            elif profile is None:
+                raise click.ClickException("No profile name provided.")
+            else:
+                params["name"] = profile
     else:
         params = {
             "name": profile,

src/optimade_launch/optimade_launch/application_state.py

@@ -6,18 +6,15 @@
 
 import click
 import docker
-from packaging.version import parse
 
 from .config import Config
-from .core import APPLICATION_ID
-from .instance import OptimadeInstance
-from .profile import Profile
+from optimade_launch.core import CONFIG_FOLDER
 from .util import get_docker_client
 from .version import __version__
 
 
 def _application_config_path():
-    return Path(click.get_app_dir(APPLICATION_ID)) / "config.toml"
+    return CONFIG_FOLDER / "config.toml"
 
 
 def _load_config():
@@ -35,33 +32,3 @@ class ApplicationState:
 
     def save_config(self):
         self.config.save(self.config_path)
-
-    # def _apply_migration_null(self):
-    #     # Since there is no config file on disk, we can assume that if at all,
-    #     # there is only the default profile present.
-    #     assert len(self.config.profiles) == 1
-    #     assert self.config.profiles[0].name == "default"
-
-    #     default_profile = self.config.profiles[0]
-    #     instance = OptimadeInstance(client=self.docker_client, profile=default_profile)
-
-    #     if instance.container:
-    #         # There is already a container present, use previously used profile.
-    #         self.config.profiles[0] = Profile.from_container(instance.container)
-
-    # def apply_migrations(self):
-    #     config_changed = False
-
-    #     # No config file saved to disk.
-    #     if not self.config_path.is_file():
-    #         self._apply_migration_null()
-    #         config_changed = True
-
-    #     # No version string stored in config.
-    #     if self.config.version != str(parse(__version__)):
-    #         self.config.version = str(parse(__version__))
-    #         config_changed = True
-
-    #     # Write any changes back to disk.
-    #     if config_changed:
-    #         self.save_config()

src/optimade_launch/optimade_launch/core.py

@@ -1,7 +1,14 @@
 # __future__ import needed for classmethod factory functions; should be dropped
 # with py 3.10.
+import os
 import logging
+import click
+from pathlib import Path
 
 APPLICATION_ID = "org.optimade.optimade_launch"
-
 LOGGER = logging.getLogger(APPLICATION_ID.split(".")[-1])
+
+if os.environ.get("OPTIMADE_LAUNCH_CONFIG_FOLDER"):
+    CONFIG_FOLDER = Path(os.environ["OPTIMADE_LAUNCH_CONFIG_FOLDER"])
+else:
+    CONFIG_FOLDER = Path(click.get_app_dir(APPLICATION_ID))

src/optimade_launch/optimade_launch/dockerfiles/Dockerfile

@@ -1,4 +1,5 @@
-FROM ghcr.io/materials-consortia/optimade:0.24.0
+ARG BASE_IMAGE
+FROM ${BASE_IMAGE}
 
 # copy repo contents and install deps
 COPY requirements.txt ./
@@ -8,4 +9,8 @@ ENV UNIX_SOCK "/tmp/gunicorn.sock"
 
 COPY run.sh /app/run.sh
 
-ENV OPTIMADE_CONFIG_FILE ""
+## Set the default config file path 
+## Then copy the config from host to container
+#ENV OPTIMADE_CONFIG_FILE "/config/config.yml"
+#
+#COPY optimade-config.yml  ${OPTIMADE_CONFIG_FILE}

src/optimade_launch/optimade_launch/instance.py

@@ -110,9 +110,13 @@ def build(self, tag: None | str = None) -> docker.models.images.Image:
         try:
             LOGGER.info(f"Building image from Dockerfile: {str(_DOCKERFILE_PATH)}")
             tag = tag or _BUILD_TAG
+            buildargs = {
+                "BASE_IMAGE": self.profile.image,
+            }
             image, logs = self.client.images.build(
                 path=str(_DOCKERFILE_PATH),
                 tag=tag,
+                buildargs=buildargs,
                 rm=True,
             )
             LOGGER.info(f"Built image: {image}")
@@ -187,11 +191,15 @@ def create(self, data: bool = False) -> Container:
             sock_filename = os.path.basename(self.profile.unix_sock)
             params_container["volumes"] = {host_sock_folder: {"bind": '/tmp', "mode": "rw"}}
             environment["UNIX_SOCK"] = f"/tmp/{sock_filename}"
+
+        # optimade config file
+        environment["OPTIMADE_CONFIG_FILE"] = self.profile.optimade_config_file
 
         if sys.platform == "linux" and "host.docker.internal" in self.profile.mongo_uri:
             params_container["extra_hosts"] = {"host.docker.internal": "host-gateway"}
 
         params_container["environment"] = environment
+
         self._container = self.client.containers.create(
             **params_container,
         )
@@ -205,7 +213,7 @@ def recreate(self) -> None:
         self.create()
 
     def start(self) -> None:
-        # TODO: check mongodb can be connected to
+        # XXX: check mongodb can be connected to
         LOGGER.info(f"Starting container '{self.profile.container_name()}'...")
         (self.container or self.create(data=True)).start()
         assert self.container is not None

src/optimade_launch/optimade_launch/profile.py

@@ -10,7 +10,7 @@
 CONTAINER_PREFIX = "optimade"
 
 DEFAULT_PORT = 8081
-DEFAULT_IMAGE = "ghcr.io/materials-consortia/optimade:0.24.0"
+DEFAULT_IMAGE = "ghcr.io/materials-consortia/optimade:0.25.2"
 DEFAULT_MONGO_URI = "mongodb://127.0.0.1:27017"
 DEFAULT_BASE_URL = "http://localhost"
 DEFAULT_INDEX_BASE_URL = "http://localhost"
@@ -35,14 +35,16 @@ def _get_configured_host_port(container: Container) -> int | None:
 @dataclass
 class Profile:
     name: str = DEFAULT_NAME
+    image: str = DEFAULT_IMAGE
     jsonl_paths: list[str] = field(default_factory=lambda: [])
     mongo_uri: str = DEFAULT_MONGO_URI
     db_name: str = "optimade"
     port: int | None = None
     unix_sock: str | None = None
-    optimade_base_url: str | None = DEFAULT_BASE_URL
-    optimade_index_base_url: str | None = DEFAULT_INDEX_BASE_URL
-    optimade_provider: str | None = DEFAULT_PROVIDER
+    optimade_config_file: str | None = None
+    optimade_base_url: str | None = None
+    optimade_index_base_url: str | None = None
+    optimade_provider: str | None = None
     optimade_validate_api_response: bool = False
 
     def __post_init__(self):
@@ -61,7 +63,7 @@ def environment(self) -> dict:
             self.mongo_uri = self.mongo_uri.replace("127.0.0.1", "host.docker.internal")
 
         return {
-            "OPTIMADE_CONFIG_FILE": None,
+            "optimade_config_file": self.optimade_config_file,
             "optimade_insert_test_data": False,
             "optimade_database_backend": "mongodb",
             "optimade_mongo_uri": self.mongo_uri,
@@ -77,7 +79,7 @@ def environment(self) -> dict:
 
     def dumps(self) -> str:
         """Dump the profile to a TOML string."""
-        return toml.dumps({k: v for k, v in asdict(self).items() if k != "name"})
+        return toml.dumps({k: v for k, v in asdict(self).items() if k != "name" and v is not None})
 
     @classmethod
     def loads(cls, name: str, s: str) -> Profile:

src/optimade_launch/pyproject.toml

@@ -22,7 +22,7 @@ dependencies = [
 ]
 
 [project.optional-dependencies]
-tests = [
+dev = [
     "pytest~=7.0.1",
     "pytest-asyncio==0.20.3",
     "pytest-mock-resources~=2.7.0",

src/optimade_launch/tests/_static/config.yaml

@@ -1,10 +1,11 @@
 ---
 name: te-st
 jsonl_paths:
-   - /var/lib/optimade-archive/te-st.jsonl
+   - /tmp/optimade.jsonl
 mongo_uri: mongodb://localhost:27017
 db_name: te-st
-unix_sock: /var/lib/optimade-sockets/te-st.sock
+unix_sock: /tmp/te-st.sock
+optimade_config_file: /tmp/optimade_config.json
 optimade_base_url: http://localhost
 optimade_index_base_url: http://localhost
 optimade_provider:

src/optimade_launch/tests/_static/optimade_config.json

@@ -0,0 +1,38 @@
+{
+    "debug": false,
+    "default_db": "test_server",
+    "base_url": "http://localhost:5000",
+    "implementation": {
+        "name": "Example implementation",
+        "source_url": "https://github.com/Materials-Consortia/optimade-python-tools",
+        "issue_tracker": "https://github.com/Materials-Consortia/optimade-python-tools/issues",
+        "maintainer": {"email": "[email protected]"}
+    },
+    "provider": {
+        "name": "Example provider",
+        "description": "Provider used for examples, not to be assigned to a real database",
+        "prefix": "exmpl",
+        "homepage": "https://example.com"
+    },
+    "index_base_url": "http://localhost:5001",
+    "provider_fields": {
+        "structures": [
+            "band_gap",
+            {"name": "chemsys", "type": "string", "description": "A string representing the chemical system in an ordered fashion"}
+        ]
+    },
+    "aliases": {
+        "structures": {
+            "id": "task_id",
+            "immutable_id": "_id",
+            "chemical_formula_descriptive": "pretty_formula",
+            "chemical_formula_reduced": "pretty_formula",
+            "chemical_formula_anonymous": "formula_anonymous"
+        }
+    },
+    "length_aliases": {
+        "structures": {
+            "chemsys": "nelements"
+        }
+    }
+}

src/optimade_launch/tests/conftest.py

@@ -96,6 +96,8 @@ async def started_instance(docker_client, monkeypatch, mongo, static_dir):
 
     if host in ("localhost", "127.0.0.1"):
         instance._container.update({"network_mode": "host"})
+
+    print(instance.profile.environment())
 
     instance.create(data=True)
     assert instance.container is not None

src/optimade_launch/tests/test_cli.py

@@ -1,8 +1,3 @@
-import logging
-from dataclasses import replace
-
-import docker
-import pytest
 from click.testing import CliRunner, Result
 
 import optimade_launch.__main__ as cli
@@ -13,9 +8,11 @@ def test_version_displays_library_version():
     """Test that the CLI displays the library version.
     """
     runner: CliRunner = CliRunner()
+
     result: Result = runner.invoke(cli.cli, ["version"])
     assert __version__ in result.output.strip(), "Version number should match library version."
     assert "Optimade Launch" in result.output.strip()
+
 
 def test_list_profiles():
     runner: CliRunner = CliRunner()

src/optimade_launch/tests/test_config.py

@@ -11,7 +11,7 @@
 
         [profiles.default]
         port = 8081
-        image = "ghcr.io/materials-consortia/optimade:0.24.0"
+        image = "ghcr.io/materials-consortia/optimade:latest"
         """
 }