Optional configuration handling improvement (#429)

- Minor adjustment so that optional configurations are requested to the
  user if they have no value in interactive mode, in non interactive
  will use the default value if any
- Log configuration messages once
- Add data purchase limit for optimization

README.md

@@ -1480,6 +1480,8 @@ Options:
                                   Update the Lean configuration file to retrieve data from the given historical provider
   --download-data                 Update the Lean configuration file to download data from the QuantConnect API, alias
                                   for --data-provider-historical QuantConnect
+  --data-purchase-limit INTEGER   The maximum amount of QCC to spend on downloading data during the backtest when using
+                                  QuantConnect as historical data provider
   --release                       Compile C# projects in release configuration instead of debug
   --image TEXT                    The LEAN engine image to use (defaults to quantconnect/lean:latest)
   --update                        Pull the LEAN engine image before running the optimizer

lean/commands/optimize.py

@@ -105,6 +105,9 @@ def get_filename_timestamp(path: Path) -> datetime:
               is_flag=True,
               default=False,
               help="Update the Lean configuration file to download data from the QuantConnect API, alias for --data-provider-historical QuantConnect")
+@option("--data-purchase-limit",
+              type=int,
+              help="The maximum amount of QCC to spend on downloading data during the backtest when using QuantConnect as historical data provider")
 @option("--release",
               is_flag=True,
               default=False,
@@ -152,6 +155,7 @@ def optimize(project: Path,
              constraint: List[str],
              data_provider_historical: Optional[str],
              download_data: bool,
+             data_purchase_limit: Optional[int],
              release: bool,
              image: Optional[str],
              update: bool,
@@ -307,6 +311,8 @@ def optimize(project: Path,
         data_provider.ensure_module_installed(organization_id)
         container.lean_config_manager.set_properties(data_provider.get_settings())
 
+    lean_config_manager.configure_data_purchase_limit(lean_config, data_purchase_limit)
+
     if not output.exists():
         output.mkdir(parents=True)
 

lean/models/json_module.py

@@ -24,6 +24,8 @@
 from copy import copy
 from abc import ABC
 
+_logged_messages = set()
+
 
 class JsonModule(ABC):
     """The JsonModule class is the base class extended for all json modules."""
@@ -178,8 +180,10 @@ def config_build(self,
             # Let's log messages for internal input configurations as well
             if configuration._log_message is not None:
                 log_message = configuration._log_message.strip()
-                if log_message:
+                if log_message and log_message not in _logged_messages:
                     logger.info(log_message)
+                    # make sure we log these messages once, we could use the same module for different functionalities
+                    _logged_messages.add(log_message)
             if type(configuration) is InternalInputUserInput:
                 continue
 
@@ -197,16 +201,18 @@ def config_build(self,
                 # NOTE: using "not" instead of "is None" because the default value can be false,
                 #       in which case we still want to prompt the user.
                 if not user_choice:
-                    if configuration._input_default != None and configuration._optional:
-                        user_choice = configuration._input_default
-                    elif interactive:
+                    if interactive:
                         default_value = configuration._input_default
                         user_choice = configuration.ask_user_for_input(default_value, logger, hide_input=hide_input)
 
                         if not isinstance(configuration, BrokerageEnvConfiguration):
                             self._save_property({f"{configuration._id}": user_choice})
                     else:
-                        missing_options.append(f"--{configuration._id}")
+                        if configuration._input_default != None and configuration._optional:
+                            # if optional and we have a default input value and the user didn't provider it we use it
+                            user_choice = configuration._input_default
+                        else:
+                            missing_options.append(f"--{configuration._id}")
 
             configuration._value = user_choice
 

tests/commands/test_live.py

@@ -318,8 +318,7 @@ def test_live_sets_dependent_configurations_from_modules_json_based_on_environme
         "ib-user-name": "trader777",
         "ib-account": "DU1234567",
         "ib-password": "hunter2",
-        "ib-enable-delayed-streaming-data": "no",
-        "ib-weekly-restart-utc-time": "22:00:00",
+        "ib-enable-delayed-streaming-data": "no"
     },
     "Tradier": {
         "tradier-account-id": "123",