From 7234308651c725ef1e4b40ad1f55c06df782c4f3 Mon Sep 17 00:00:00 2001
From: Fivele-Li <128388363+Fivele-Li@users.noreply.github.com>
Date: Fri, 12 May 2023 17:35:37 +0800
Subject: [PATCH] Add base config in yml (#1500)

* path on Windows contains double '/' which may cause open file failed.

* locate import numpy error

* locate import numpy error

* locate import numpy error

* locate import numpy error

* locate import numpy error

* locate import numpy error

* locate import numpy error

* locate import numpy error

* locate import numpy error

* locate import numpy error

* add baseConfig in yml,user can add new keys or update/drop keys in baseConfig;

* locate import numpy error

* locate import numpy error

* locate import numpy error

* locate import numpy error

* locate import numpy error

* locate import numpy error

* locate import numpy error

* pip release version 23.1 on Apr.15 2023, CI failed to run, Please refer to #1495 ofr detailed logs. The pip version has been temporarily fixed to 23.0.1.

* 1.Search for baseConfig in multiple directories;
2.Add user instructions in qrun;

* fix format with black

* 1.modify baseConfig key to BASE_CONFIG_PATH;
2.only find config file in absolute path and relative path;

* load BASE_CONFIG_PATH on absolute path & relative path;

* fix Lint with black

---------

Co-authored-by: lijinhui <362237642@qq.com>
---
 qlib/workflow/cli.py | 45 +++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 44 insertions(+), 1 deletion(-)

diff --git a/qlib/workflow/cli.py b/qlib/workflow/cli.py
index 25239d1f10..c2265ea5db 100644
--- a/qlib/workflow/cli.py
+++ b/qlib/workflow/cli.py
@@ -1,6 +1,6 @@
 #  Copyright (c) Microsoft Corporation.
 #  Licensed under the MIT License.
-
+import logging
 import sys
 import os
 from pathlib import Path
@@ -10,6 +10,12 @@
 import ruamel.yaml as yaml
 from qlib.config import C
 from qlib.model.trainer import task_train
+from qlib.utils.data import update_config
+from qlib.log import get_module_logger
+from qlib.utils import set_log_with_config
+
+set_log_with_config(C.logging_config)
+logger = get_module_logger("qrun", logging.INFO)
 
 
 def get_path_list(path):
@@ -47,10 +53,47 @@ def workflow(config_path, experiment_name="workflow", uri_folder="mlruns"):
     This is a Qlib CLI entrance.
     User can run the whole Quant research workflow defined by a configure file
     - the code is located here ``qlib/workflow/cli.py`
+
+    User can specify a base_config file in your workflow.yml file by adding "BASE_CONFIG_PATH".
+    Qlib will load the configuration in BASE_CONFIG_PATH first, and the user only needs to update the custom fields
+    in their own workflow.yml file.
+
+    For examples:
+
+        qlib_init:
+            provider_uri: "~/.qlib/qlib_data/cn_data"
+            region: cn
+        BASE_CONFIG_PATH: "workflow_config_lightgbm_Alpha158_csi500.yaml"
+        market: csi300
+
     """
     with open(config_path) as fp:
         config = yaml.safe_load(fp)
 
+    base_config_path = config.get("BASE_CONFIG_PATH", None)
+    if base_config_path:
+        logger.info(f"Use BASE_CONFIG_PATH: {base_config_path}")
+        base_config_path = Path(base_config_path)
+
+        # it will find config file in absolute path and relative path
+        if base_config_path.exists():
+            path = base_config_path
+        else:
+            logger.info(
+                f"Can't find BASE_CONFIG_PATH base on: {Path.cwd()}, "
+                f"try using relative path to config path: {Path(config_path).absolute()}"
+            )
+            relative_path = Path(config_path).absolute().parent.joinpath(base_config_path)
+            if relative_path.exists():
+                path = relative_path
+            else:
+                raise FileNotFoundError(f"Can't find the BASE_CONFIG file: {base_config_path}")
+
+        with open(path) as fp:
+            base_config = yaml.safe_load(fp)
+        logger.info(f"Load BASE_CONFIG_PATH succeed: {path.resolve()}")
+        config = update_config(base_config, config)
+
     # config the `sys` section
     sys_config(config, config_path)