From a491905da20935b61685f0fee3642cc743b02895 Mon Sep 17 00:00:00 2001 From: Martin Molinero Date: Fri, 6 Dec 2024 19:55:22 -0300 Subject: [PATCH] Minor private cloud api improvements --- .github/workflows/build.yml | 8 +-- README.md | 59 +++++++++++++++++----- lean/commands/private_cloud/__init__.py | 2 + lean/commands/private_cloud/add_compute.py | 53 +++++++++++++++++++ lean/commands/private_cloud/start.py | 40 ++++++++++++--- 5 files changed, 137 insertions(+), 25 deletions(-) create mode 100644 lean/commands/private_cloud/add_compute.py diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 006ecc9b..38933946 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -8,7 +8,7 @@ jobs: strategy: matrix: - os: [ubuntu-22.04, ubuntu-20.04, macos-12, macos-13, windows-latest] + os: [ubuntu-22.04, ubuntu-20.04, macos-13, windows-latest] python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] steps: @@ -63,7 +63,7 @@ jobs: strategy: matrix: - os: [ubuntu-22.04, macos-12, windows-latest] + os: [ubuntu-22.04, macos-13, windows-latest] python-version: ["3.10"] # on a new tag (release), after 'test' and 'release' have ended we will run this @@ -96,7 +96,7 @@ jobs: shell: bash - name: Create Portable Macos - if: matrix.os == 'macos-12' || matrix.os == 'macos-11' + if: matrix.os == 'macos-13' env: MACOS_CERTIFICATE: ${{ secrets.MACOS_CERTIFICATE }} MACOS_CERTIFICATE_PWD: ${{ secrets.MACOS_CERTIFICATE_PWD }} @@ -123,7 +123,7 @@ jobs: shell: bash - name: Zip package and notarize Macos - if: matrix.os == 'macos-12' + if: matrix.os == 'macos-13' env: APPLE_ID: ${{ secrets.APPLE_ID }} APPLE_TEAM_ID: ${{ secrets.APPLE_TEAM_ID }} diff --git a/README.md b/README.md index ff191292..91abe5f0 100644 --- a/README.md +++ b/README.md @@ -118,6 +118,7 @@ A locally-focused workflow (local development, local execution) with the CLI may - [`lean object-store properties`](#lean-object-store-properties) - [`lean object-store set`](#lean-object-store-set) - [`lean optimize`](#lean-optimize) +- [`lean private-cloud add-compute`](#lean-private-cloud-add-compute) - [`lean private-cloud start`](#lean-private-cloud-start) - [`lean private-cloud stop`](#lean-private-cloud-stop) - [`lean project-create`](#lean-project-create) @@ -1818,6 +1819,34 @@ Options: _See code: [lean/commands/optimize.py](lean/commands/optimize.py)_ +### `lean private-cloud add-compute` + +Add private cloud compute + +``` +Usage: lean private-cloud add-compute [OPTIONS] + + Add private cloud compute + +Options: + --token TEXT The master server token + --master-domain, --master-ip TEXT + The master server domain + --master-port INTEGER The master server port + --slave-domain, --slave-ip TEXT + The slave server domain + --update Pull the latest image before starting + --no-update Do not update to the latest version + --compute TEXT Compute configuration to use + --extra-docker-config TEXT Extra docker configuration as a JSON string + --stop Stop any existing deployment + --lean-config FILE The Lean configuration file that should be used (defaults to the nearest lean.json) + --verbose Enable debug logging + --help Show this message and exit. +``` + +_See code: [lean/commands/private_cloud/add_compute.py](lean/commands/private_cloud/add_compute.py)_ + ### `lean private-cloud start` Start a new private cloud @@ -1828,20 +1857,22 @@ Usage: lean private-cloud start [OPTIONS] Start a new private cloud Options: - --master Run in master mode - --slave Run in slave mode - --token TEXT The master server token - --master-domain TEXT The master server domain - --master-port INTEGER The master server port - --slave-domain TEXT The slave server domain - --update Pull the latest image before starting - --no-update Do not update to the latest version - --compute TEXT Compute configuration to use - --extra-docker-config TEXT Extra docker configuration as a JSON string - --stop Stop any existing deployment - --lean-config FILE The Lean configuration file that should be used (defaults to the nearest lean.json) - --verbose Enable debug logging - --help Show this message and exit. + --master Run in master mode + --slave Run in slave mode + --token TEXT The master server token + --master-domain, --master-ip TEXT + The master server domain + --master-port INTEGER The master server port + --slave-domain, --slave-ip TEXT + The slave server domain + --update Pull the latest image before starting + --no-update Do not update to the latest version + --compute TEXT Compute configuration to use + --extra-docker-config TEXT Extra docker configuration as a JSON string + --stop Stop any existing deployment + --lean-config FILE The Lean configuration file that should be used (defaults to the nearest lean.json) + --verbose Enable debug logging + --help Show this message and exit. ``` _See code: [lean/commands/private_cloud/start.py](lean/commands/private_cloud/start.py)_ diff --git a/lean/commands/private_cloud/__init__.py b/lean/commands/private_cloud/__init__.py index 2f042bfd..b154688c 100644 --- a/lean/commands/private_cloud/__init__.py +++ b/lean/commands/private_cloud/__init__.py @@ -15,6 +15,7 @@ from lean.commands.private_cloud.start import start from lean.commands.private_cloud.stop import stop +from lean.commands.private_cloud.add_compute import add_compute @group() @@ -27,3 +28,4 @@ def private_cloud() -> None: private_cloud.add_command(start) private_cloud.add_command(stop) +private_cloud.add_command(add_compute) diff --git a/lean/commands/private_cloud/add_compute.py b/lean/commands/private_cloud/add_compute.py new file mode 100644 index 00000000..65d8efd0 --- /dev/null +++ b/lean/commands/private_cloud/add_compute.py @@ -0,0 +1,53 @@ +# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. +# Lean CLI v1.0. Copyright 2021 QuantConnect Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import Optional + +from click import option, command + +from lean.click import LeanCommand +from lean.commands.private_cloud.start import start_command + + +@command(cls=LeanCommand, requires_lean_config=True, requires_docker=True, help="Add private cloud compute") +@option("--token", type=str, required=False, help="The master server token") +@option("--master-domain", "--master-ip", type=str, required=False, help="The master server domain") +@option("--master-port", type=int, required=False, default=443, help="The master server port") +@option("--slave-domain", "--slave-ip", type=str, required=False, help="The slave server domain") +@option("--update", is_flag=True, default=False, help="Pull the latest image before starting") +@option("--no-update", is_flag=True, default=False, help="Do not update to the latest version") +@option("--compute", type=str, required=False, help="Compute configuration to use") +@option("--extra-docker-config", type=str, default="{}", help="Extra docker configuration as a JSON string") +@option("--image", type=str, hidden=True) +@option("--stop", is_flag=True, default=False, help="Stop any existing deployment") +def add_compute(token: str, + master_domain: str, + slave_domain: str, + master_port: int, + update: bool, + no_update: bool, + compute: Optional[str], + extra_docker_config: Optional[str], + image: Optional[str], + stop: bool) -> None: + start_command(False, + True, + token, + master_domain, + slave_domain, + master_port, + update, + no_update, + compute, + extra_docker_config, + image, + stop) diff --git a/lean/commands/private_cloud/start.py b/lean/commands/private_cloud/start.py index bcaf090f..145e360b 100644 --- a/lean/commands/private_cloud/start.py +++ b/lean/commands/private_cloud/start.py @@ -15,7 +15,7 @@ from typing import Optional from json import loads -from click import command, option +from click import command, option, prompt from docker.errors import APIError from docker.types import Mount @@ -106,9 +106,9 @@ def get_ip_address(): @option("--master", is_flag=True, default=False, help="Run in master mode") @option("--slave", is_flag=True, default=False, help="Run in slave mode") @option("--token", type=str, required=False, help="The master server token") -@option("--master-domain", type=str, required=False, help="The master server domain") +@option("--master-domain", "--master-ip", type=str, required=False, help="The master server domain") @option("--master-port", type=int, required=False, default=443, help="The master server port") -@option("--slave-domain", type=str, required=False, help="The slave server domain") +@option("--slave-domain", "--slave-ip", type=str, required=False, help="The slave server domain") @option("--update", is_flag=True, default=False, help="Pull the latest image before starting") @option("--no-update", is_flag=True, default=False, help="Do not update to the latest version") @option("--compute", type=str, required=False, help="Compute configuration to use") @@ -127,6 +127,32 @@ def start(master: bool, extra_docker_config: Optional[str], image: Optional[str], stop: bool) -> None: + start_command(master, + slave, + token, + master_domain, + slave_domain, + master_port, + update, + no_update, + compute, + extra_docker_config, + image, + stop) + + +def start_command(master: bool, + slave: bool, + token: str, + master_domain: str, + slave_domain: str, + master_port: int, + update: bool, + no_update: bool, + compute: Optional[str], + extra_docker_config: Optional[str], + image: Optional[str], + stop: bool) -> None: logger = container.logger if stop: @@ -139,8 +165,7 @@ def start(master: bool, slave = True if not master_domain: - master_domain = get_ip_address() - logger.info(f"'--master-domain' was not provided using '{master_domain}'") + master_domain = prompt("Master domain", get_ip_address()) str_mode = 'slave' if slave else 'master' logger.info(f'Start running in {str_mode} mode') @@ -157,7 +182,7 @@ def start(master: bool, if slave: if not token: - raise RuntimeError(f"Master token '--token' is required when running as slave") + token = prompt("Master token") else: if not token: from uuid import uuid4 @@ -199,7 +224,8 @@ def start(master: bool, try: from requests import get resp = get(f'http://{master_domain}:{master_port}', stream=True) - slave_domain = resp.raw._connection.sock.getsockname()[0] + potential_slave_domain = resp.raw._connection.sock.getsockname()[0] + slave_domain = prompt("Slave domain", potential_slave_domain) break except Exception as e: from time import sleep