Remove hardcoded python and dotnet version (#441)

* Remove hardcoded python and dotnet version

- Remove hardcoded python and dotnet version, these will be retrieved
  from the docker labels of the image being used

* Move default python dotnet values to constant

lean/commands/library/add.py

@@ -13,10 +13,10 @@
 
 from pathlib import Path
 from typing import Any, Dict, Optional, Tuple
-from lean.constants import LEAN_STRICT_PYTHON_VERSION
 from click import command, argument, option
 
 from lean.click import LeanCommand, PathParameter
+from lean.constants import DEFAULT_LEAN_STRICT_PYTHON_VERSION
 from lean.container import container
 from lean.models.errors import MoreInfoError
 
@@ -124,7 +124,7 @@ def _is_pypi_file_compatible(file: Dict[str, Any], required_python_version) -> b
     return True
 
 
-def _get_pypi_package(name: str, version: Optional[str]) -> Tuple[str, str]:
+def _get_pypi_package(name: str, version: Optional[str], python_version: str) -> Tuple[str, str]:
     """Retrieves the properly-capitalized name and the latest compatible version of a package from PyPI.
 
     If the version is already given, this method checks whether that version is compatible with the Docker images.
@@ -148,7 +148,7 @@ def _get_pypi_package(name: str, version: Optional[str]) -> Tuple[str, str]:
     pypi_data = loads(response.text)
     name = pypi_data["info"]["name"]
 
-    required_python_version = StrictVersion(LEAN_STRICT_PYTHON_VERSION)
+    required_python_version = StrictVersion(python_version)
 
     last_compatible_version = None
     last_compatible_version_upload_time = None
@@ -237,7 +237,12 @@ def _add_pypi_package_to_python_project(project_dir: Path, name: str, version: O
     else:
         logger.info("Retrieving latest compatible version from PyPI")
 
-    name, version = _get_pypi_package(name, version)
+    project_config = container.project_config_manager.get_project_config(project_dir)
+    engine_image = container.cli_config_manager.get_engine_image(project_config.get("engine-image", None))
+    python_version = container.docker_manager.get_image_label(engine_image, 'strict_python_version',
+                                                              DEFAULT_LEAN_STRICT_PYTHON_VERSION)
+
+    name, version = _get_pypi_package(name, version, python_version)
 
     requirements_file = project_dir / "requirements.txt"
     logger.info(f"Adding {name} {version} to '{path_manager.get_relative_path(requirements_file)}'")

lean/commands/optimize.py

@@ -336,7 +336,10 @@ def optimize(project: Path,
     build_and_configure_modules(addon_module, cli_addon_modules, organization_id, lean_config,
                                 kwargs, logger, environment_name)
 
-    run_options = lean_runner.get_basic_docker_config(lean_config, algorithm_file, output, None, release, should_detach)
+    container.update_manager.pull_docker_image_if_necessary(engine_image, update, no_update)
+
+    run_options = lean_runner.get_basic_docker_config(lean_config, algorithm_file, output, None, release, should_detach,
+                                                      engine_image)
 
     run_options["working_dir"] = "/Lean/Optimizer.Launcher/bin/Debug"
     run_options["commands"].append(f"dotnet QuantConnect.Optimizer.Launcher.dll{' --estimate' if estimate else ''}")
@@ -346,7 +349,6 @@ def optimize(project: Path,
               type="bind",
               read_only=True)
     )
-    container.update_manager.pull_docker_image_if_necessary(engine_image, update, no_update)
 
     # Add known additional run options from the extra docker config
     LeanRunner.parse_extra_docker_config(run_options, loads(extra_docker_config))

lean/commands/research.py

@@ -128,12 +128,24 @@ def research(project: Path,
     for key, value in extra_config:
         lean_config[key] = value
 
+    project_config_manager = container.project_config_manager
+    cli_config_manager = container.cli_config_manager
+
+    project_config = project_config_manager.get_project_config(algorithm_file.parent)
+    research_image = cli_config_manager.get_research_image(image or project_config.get("research-image", None))
+
+    container.update_manager.pull_docker_image_if_necessary(research_image, update, no_update)
+
+    if str(research_image) != DEFAULT_RESEARCH_IMAGE:
+        logger.warn(f'A custom research image: "{research_image}" is being used!')
+
     run_options = lean_runner.get_basic_docker_config(lean_config,
                                                       algorithm_file,
                                                       temp_manager.create_temporary_directory(),
                                                       None,
                                                       False,
-                                                      detach)
+                                                      detach,
+                                                      research_image)
 
     # Mount the config in the notebooks directory as well
     local_config_path = next(m["Source"] for m in run_options["mounts"] if m["Target"].endswith("config.json"))
@@ -174,17 +186,6 @@ def research(project: Path,
     # Add known additional run options from the extra docker config
     LeanRunner.parse_extra_docker_config(run_options, loads(extra_docker_config))
 
-    project_config_manager = container.project_config_manager
-    cli_config_manager = container.cli_config_manager
-
-    project_config = project_config_manager.get_project_config(algorithm_file.parent)
-    research_image = cli_config_manager.get_research_image(image or project_config.get("research-image", None))
-
-    if str(research_image) != DEFAULT_RESEARCH_IMAGE:
-        logger.warn(f'A custom research image: "{research_image}" is being used!')
-
-    container.update_manager.pull_docker_image_if_necessary(research_image, update, no_update)
-
     try:
         container.docker_manager.run_image(research_image, **run_options)
     except APIError as error:

lean/components/docker/docker_manager.py

@@ -38,6 +38,16 @@ def __init__(self, logger: Logger, temp_manager: TempManager, platform_manager:
         self._temp_manager = temp_manager
         self._platform_manager = platform_manager
 
+    def get_image_label(self, image: DockerImage, label: str, default: str) -> str:
+        docker_image = self._get_docker_client().images.get(image.name)
+
+        for name, value in docker_image.labels.items():
+            if name == label:
+                self._logger.debug(f"Label '{label}' found in image '{image.name}', value {value}")
+                return value
+        self._logger.info(f"Label '{label}' not found in image '{image.name}', using default {default}")
+        return default
+
     def pull_image(self, image: DockerImage) -> None:
         """Pulls a Docker image.
 
@@ -563,13 +573,13 @@ def _format_source_path(self, path: str) -> str:
                 break
 
         return path
-    
+
     def get_container_port(self, container_name: str, internal_port: str) -> Optional[int]:
         """
         Returns a containers external port for a mapped internal port
         :param container_name: Name of the container
         :param internal_port: The internal port of container. If protocol not included
-        we assume /tcp. ex. 5678/tcp 
+        we assume /tcp. ex. 5678/tcp
         :return: The external port that is linked to it, or None if it does not exist
         """
 

lean/components/docker/lean_runner.py

@@ -23,7 +23,8 @@
 from lean.components.util.project_manager import ProjectManager
 from lean.components.util.temp_manager import TempManager
 from lean.components.util.xml_manager import XMLManager
-from lean.constants import MODULES_DIRECTORY, TERMINAL_LINK_PRODUCT_ID, LEAN_ROOT_PATH, DEFAULT_DATA_DIRECTORY_NAME
+from lean.constants import MODULES_DIRECTORY, LEAN_ROOT_PATH, DEFAULT_DATA_DIRECTORY_NAME, \
+    DEFAULT_LEAN_DOTNET_FRAMEWORK, DEFAULT_LEAN_PYTHON_VERSION
 from lean.constants import DOCKER_PYTHON_SITE_PACKAGES_PATH
 from lean.models.docker import DockerImage
 from lean.models.utils import DebuggingMethod
@@ -97,7 +98,8 @@ def run_lean(self,
                                                    output_dir,
                                                    debugging_method,
                                                    release,
-                                                   detach)
+                                                   detach,
+                                                   image)
 
         # Add known additional run options from the extra docker config
         self.parse_extra_docker_config(run_options, extra_docker_config)
@@ -175,7 +177,8 @@ def get_basic_docker_config(self,
                                 output_dir: Path,
                                 debugging_method: Optional[DebuggingMethod],
                                 release: bool,
-                                detach: bool) -> Dict[str, Any]:
+                                detach: bool,
+                                image: DockerImage) -> Dict[str, Any]:
         """Creates a basic Docker config to run the engine with.
 
         This method constructs the parts of the Docker config that is the same for both the engine and the optimizer.
@@ -186,6 +189,7 @@ def get_basic_docker_config(self,
         :param debugging_method: the debugging method if debugging needs to be enabled, None if not
         :param release: whether C# projects should be compiled in release configuration instead of debug
         :param detach: whether LEAN should run in a detached container
+        :param image: The docker image that will be used
         :return: the Docker configuration containing basic configuration to run Lean
         """
         from docker.types import Mount
@@ -309,7 +313,10 @@ def get_basic_docker_config(self,
             # Create a C# project used to resolve the dependencies of the modules
             run_options["commands"].append("mkdir /ModulesProject")
             run_options["commands"].append("dotnet new sln -o /ModulesProject")
-            run_options["commands"].append("dotnet new classlib -o /ModulesProject -f net6.0 --no-restore")
+
+            framework_ver = self._docker_manager.get_image_label(image, 'target_framework',
+                                                                 DEFAULT_LEAN_DOTNET_FRAMEWORK)
+            run_options["commands"].append(f"dotnet new classlib -o /ModulesProject -f {framework_ver} --no-restore")
             run_options["commands"].append("rm /ModulesProject/Class1.cs")
 
             # Add all modules to the project, automatically resolving all dependencies
@@ -325,7 +332,8 @@ def get_basic_docker_config(self,
 
         # Set up language-specific run options
         self.setup_language_specific_run_options(run_options, project_dir, algorithm_file,
-                                            set_up_common_csharp_options_called, release)
+                                                 set_up_common_csharp_options_called, release,
+                                                 image)
 
         # Save the final Lean config to a temporary file so we can mount it into the container
         config_path = self._temp_manager.create_temporary_directory() / "config.json"
@@ -359,11 +367,12 @@ def get_basic_docker_config(self,
 
         return run_options
 
-    def set_up_python_options(self, project_dir: Path, run_options: Dict[str, Any]) -> None:
+    def set_up_python_options(self, project_dir: Path, run_options: Dict[str, Any], image: DockerImage) -> None:
         """Sets up Docker run options specific to Python projects.
 
         :param project_dir: the path to the project directory
         :param run_options: the dictionary to append run options to
+        :param image: the docker image that will be used
         """
 
         from docker.types import Mount
@@ -410,10 +419,14 @@ def set_up_python_options(self, project_dir: Path, run_options: Dict[str, Any])
             "mode": "rw"
         }
 
+        python_version = self._docker_manager.get_image_label(image, 'python_version',
+                                                              DEFAULT_LEAN_PYTHON_VERSION)
+        site_packages_path = DOCKER_PYTHON_SITE_PACKAGES_PATH.replace('{LEAN_PYTHON_VERSION}', python_version)
+
         # Mount a volume to the user packages directory so we don't install packages every time
         site_packages_volume = self._docker_manager.create_site_packages_volume(requirements_txt)
         run_options["volumes"][site_packages_volume] = {
-            "bind": f"{DOCKER_PYTHON_SITE_PACKAGES_PATH}",
+            "bind": f"{site_packages_path}",
             "mode": "rw"
         }
 
@@ -424,7 +437,7 @@ def set_up_python_options(self, project_dir: Path, run_options: Dict[str, Any])
         # We only need to do this if it hasn't already been done before for this site packages volume
         # To keep track of this we create a special file in the site packages directory after installation
         # If this file already exists we can skip pip install completely
-        marker_file = f"{DOCKER_PYTHON_SITE_PACKAGES_PATH}/pip-install-done"
+        marker_file = f"{site_packages_path}/pip-install-done"
         run_options["commands"].extend([
             f"! test -f {marker_file} && pip install --user --progress-bar off -r /requirements.txt",
             f"touch {marker_file}"
@@ -452,12 +465,14 @@ def _concat_python_requirements(self, requirements_files: List[Path]) -> str:
         requirements = sorted(set(requirements))
         return "\n".join(requirements)
 
-    def set_up_csharp_options(self, project_dir: Path, run_options: Dict[str, Any], release: bool) -> None:
+    def set_up_csharp_options(self, project_dir: Path, run_options: Dict[str, Any], release: bool,
+                              image: DockerImage) -> None:
         """Sets up Docker run options specific to C# projects.
 
         :param project_dir: the path to the project directory
         :param run_options: the dictionary to append run options to
         :param release: whether C# projects should be compiled in release configuration instead of debug
+        :param image: the docker image that will be used
         """
         compile_root = self._get_csharp_compile_root(project_dir)
 
@@ -474,11 +489,13 @@ def set_up_csharp_options(self, project_dir: Path, run_options: Dict[str, Any],
         for path in compile_root.rglob("*.csproj"):
             self._ensure_csproj_uses_correct_lean(compile_root, path, csproj_temp_dir, run_options)
 
+        framework_ver = self._docker_manager.get_image_label(image, 'target_framework',
+                                                             DEFAULT_LEAN_DOTNET_FRAMEWORK)
         # Set up the MSBuild properties
         msbuild_properties = {
             "Configuration": "Release" if release else "Debug",
             "Platform": "AnyCPU",
-            "TargetFramework": "net6.0",
+            "TargetFramework": framework_ver,
             "OutputPath": "/Compile/bin",
             "GenerateAssemblyInfo": "false",
             "GenerateTargetFrameworkAttribute": "false",
@@ -722,14 +739,14 @@ def _force_disk_provider_if_necessary(self,
             lean_config[config_key] = disk_provider
 
     def setup_language_specific_run_options(self, run_options, project_dir, algorithm_file,
-                                            set_up_common_csharp_options_called, release) -> None:
+                                            set_up_common_csharp_options_called, release, image: DockerImage) -> None:
         # Set up language-specific run options
         if algorithm_file.name.endswith(".py"):
-            self.set_up_python_options(project_dir, run_options)
+            self.set_up_python_options(project_dir, run_options, image)
         else:
             if not set_up_common_csharp_options_called:
                 self.set_up_common_csharp_options(run_options)
-            self.set_up_csharp_options(project_dir, run_options, release)
+            self.set_up_csharp_options(project_dir, run_options, release, image)
 
     def format_error_before_logging(self, chunk: str):
         from lean.components.util import compiler