Fix Local Live Deploy Using Environment Option (#399)

* update essential properties when loading from envrionment

* review

* handle null case

* auto fill essential properties

* add test

Author: rjra2611

lean/commands/live/deploy.py

@@ -21,14 +21,14 @@
 from lean.models.errors import MoreInfoError
 from lean.models.lean_config_configurer import LeanConfigConfigurer
 from lean.models.logger import Option
-from lean.models.configuration import InternalInputUserInput
+from lean.models.configuration import ConfigurationsEnvConfiguration, InternalInputUserInput
 from lean.models.click_options import options_from_json, get_configs_for_options
 from lean.models.json_module import LiveInitialStateInput
 from lean.commands.live.live import live
 from lean.components.util.live_utils import get_last_portfolio_cash_holdings, configure_initial_cash_balance, configure_initial_holdings,\
                                             _configure_initial_cash_interactively, _configure_initial_holdings_interactively
 from lean.models.data_providers import all_data_providers
-from lean.components.util.json_modules_handler import build_and_configure_modules, get_and_build_module
+from lean.components.util.json_modules_handler import build_and_configure_modules, get_and_build_module, update_essential_properties_available
 
 _environment_skeleton = {
     "live-mode": True,
@@ -54,11 +54,20 @@ def _get_configurable_modules_from_environment(lean_config: Dict[str, Any], envi
 
     brokerage = environment["live-mode-brokerage"]
     data_queue_handlers = environment["data-queue-handler"]
-    [brokerage_configurer] = [local_brokerage for local_brokerage in all_local_brokerages if local_brokerage.get_live_name(environment_name) == brokerage]
-    data_feed_configurers = [local_data_feed for local_data_feed in all_local_data_feeds if local_data_feed.get_live_name(environment_name) in data_queue_handlers]
+    [brokerage_configurer] = [local_brokerage for local_brokerage in all_local_brokerages if _get_brokerage_base_name(local_brokerage.get_live_name(environment_name)) == _get_brokerage_base_name(brokerage)]
+    data_queue_handlers_base_names = [_get_brokerage_base_name(data_queue_handler) for data_queue_handler in data_queue_handlers]
+    data_feed_configurers = [local_data_feed for local_data_feed in all_local_data_feeds if _get_brokerage_base_name(local_data_feed.get_live_name(environment_name)) in data_queue_handlers_base_names]
     return brokerage_configurer, data_feed_configurers
 
 
+def _get_brokerage_base_name(brokerage: str) -> str:
+    """Returns the base name of the brokerage.
+
+    :param brokerage: the name of the brokerage
+    :return: the base name of the brokerage
+    """
+    return brokerage.split('.')[-1]
+
 def _install_modules(modules: List[LeanConfigConfigurer], user_kwargs: Dict[str, Any]) -> None:
     """Raises an error if any of the given modules are not installed.
 
@@ -326,6 +335,52 @@ def deploy(project: Path,
     if environment is not None:
         environment_name = environment
         lean_config = lean_config_manager.get_complete_lean_config(environment_name, algorithm_file, None)
+        
+        lean_environment = lean_config["environments"][environment_name]
+        for key in ["live-mode-brokerage", "data-queue-handler"]:
+            if key not in lean_environment:
+                raise MoreInfoError(f"The '{environment_name}' environment does not specify a {key}",
+                                    "https://www.lean.io/docs/v2/lean-cli/live-trading/algorithm-control")
+
+        brokerage = lean_environment["live-mode-brokerage"]
+        data_queue_handlers = lean_environment["data-queue-handler"]
+        data_queue_handlers_base_names = [_get_brokerage_base_name(data_queue_handler) for data_queue_handler in data_queue_handlers]
+        data_feed_configurers = []
+
+        for local_brokerage in all_local_brokerages:
+            configuration_environments: List[ConfigurationsEnvConfiguration] = [config for config in local_brokerage._lean_configs if config._is_type_configurations_env]
+            for configuration_environment in configuration_environments:
+                configuration_environment_values = list(configuration_environment._env_and_values.values())[0]
+                if any(True for x in configuration_environment_values if x["name"] == "live-mode-brokerage" and _get_brokerage_base_name(x["value"]) == _get_brokerage_base_name(brokerage)):
+                    brokerage_configurer = local_brokerage
+                    # fill essential properties
+                    for condition in configuration_environment._filter._conditions:
+                        if condition._type != "exact-match":
+                            continue
+                        property_name_to_fill = local_brokerage.convert_lean_key_to_variable(condition._dependent_config_id)
+                        property_value_to_fill = condition._pattern
+                        kwargs[property_name_to_fill] = property_value_to_fill
+                        lean_config[condition._dependent_config_id] = property_value_to_fill
+                    break
+        
+        for local_data_feed in all_local_data_feeds:
+            configuration_environments: List[ConfigurationsEnvConfiguration] = [config for config in local_data_feed._lean_configs if config._is_type_configurations_env]
+            for configuration_environment in configuration_environments:
+                configuration_environment_values = list(configuration_environment._env_and_values.values())[0]
+                if any(True for x in configuration_environment_values if x["name"] == "data-queue-handler" and _get_brokerage_base_name(x["value"]) in data_queue_handlers_base_names):
+                    data_feed_configurers.append(local_data_feed)
+                    # fill essential properties
+                    for condition in configuration_environment._filter._conditions:
+                        if condition._type != "exact-match":
+                            continue
+                        property_name_to_fill = local_data_feed.convert_lean_key_to_variable(condition._dependent_config_id)
+                        property_value_to_fill = condition._pattern
+                        kwargs[property_name_to_fill] = property_value_to_fill
+                        lean_config[condition._dependent_config_id] = property_value_to_fill
+
+        [update_essential_properties_available([brokerage_configurer], kwargs)]
+        [update_essential_properties_available(data_feed_configurers, kwargs)]
+
     elif brokerage is not None or len(data_feed) > 0:
         ensure_options(["brokerage", "data_feed"])
 

lean/components/util/json_modules_handler.py

@@ -56,3 +56,12 @@ def get_and_build_module(target_module_name: str, module_list: List[JsonModule],
     target_module.update_configs(required_properties_value)
     logger.debug(f"json_module_handler.get_and_build_module(): non-interactive: required_properties_value with module {target_module_name}: {required_properties_value}")
     return target_module
+
+
+def update_essential_properties_available(module_list: List[JsonModule], properties: Dict[str, Any]) -> JsonModule:
+    for target_module in module_list:
+        # update essential properties from brokerage to datafeed
+        # needs to be updated before fetching required properties
+        essential_properties = [target_module.convert_lean_key_to_variable(prop) for prop in target_module.get_essential_properties()]
+        essential_properties_value = {target_module.convert_variable_to_lean_key(prop) : properties[prop] for prop in essential_properties if properties[prop] is not None}
+        target_module.update_configs(essential_properties_value)

lean/models/json_module.py

@@ -91,8 +91,8 @@ def get_configurations_env_values_from_name(self, target_env: str) -> List[Dict[
                         config._is_type_configurations_env and self.check_if_config_passes_filters(
                             config)
                         ] or [None]
-        if env_config is not None and target_env in env_config._env_and_values.keys():
-            env_config_values = env_config._env_and_values[target_env]
+        if env_config is not None:
+            env_config_values = list(env_config._env_and_values.values())[0]
         return env_config_values
 
     def get_config_from_type(self, config_type: Configuration) -> str:

tests/commands/test_live.py

@@ -65,6 +65,34 @@ def create_fake_environment(name: str, live_mode: bool) -> None:
     path.write_text(config, encoding="utf-8")
 
 
+def create_fake_binance_environment(name: str, live_mode: bool) -> None:
+    path = Path.cwd() / "lean.json"
+    config = path.read_text(encoding="utf-8")
+    config = config.replace("{", f"""
+{{
+    "binance-use-testnet": "live",
+    "binance-exchange-name": "binance",
+    "binance-api-secret": "abc",
+    "binance-api-key": "abc",
+    "organization-id": "abc",
+
+    "environments": {{
+        "{name}": {{
+            "live-mode": {str(live_mode).lower()},
+
+            "live-mode-brokerage": "QuantConnect.BinanceBrokerage.BinanceCoinFuturesBrokerage",
+            "data-queue-handler": [ "QuantConnect.BinanceBrokerage.BinanceCoinFuturesBrokerage" ],
+            "setup-handler": "QuantConnect.Lean.Engine.Setup.BrokerageSetupHandler",
+            "result-handler": "QuantConnect.Lean.Engine.Results.LiveTradingResultHandler",
+            "data-feed-handler": "QuantConnect.Lean.Engine.DataFeeds.LiveTradingDataFeed",
+            "real-time-handler": "QuantConnect.Lean.Engine.RealTime.LiveTradingRealTimeHandler",
+            "transaction-handler": "QuantConnect.Lean.Engine.TransactionHandlers.BrokerageTransactionHandler",
+            "history-provider": [ "BrokerageHistoryProvider", "SubscriptionDataReaderHistoryProvider" ]
+        }}
+    }},
+    """)
+
+    path.write_text(config, encoding="utf-8")
 
 def test_live_calls_lean_runner_with_correct_algorithm_file() -> None:
     # TODO: currently it is not using the live-paper envrionment
@@ -258,6 +286,20 @@ def test_live_aborts_when_lean_config_is_missing_properties(target: str, replace
 
     lean_runner.run_lean.assert_not_called()
 
+def test_live_sets_dependent_configurations_from_modules_json_based_on_environment() -> None:
+    create_fake_lean_cli_directory()
+    create_fake_binance_environment("live-binance", True)
+    lean_runner = container.lean_runner
+
+    config_path = Path.cwd() / "lean.json"
+    config = config_path.read_text(encoding="utf-8")
+    config_path.write_text(config.replace("binance-exchange-name", "different-config"), encoding="utf-8")
+
+    result = CliRunner().invoke(lean, ["live", "Python Project", "--environment", "live-binance"])
+
+    assert result.exit_code == 0
+
+    lean_runner.run_lean.assert_called()
 
 terminal_link_required_options = {
     "terminal-link-connection-type": "SAPI",