Merge pull request #132 from rjra2611/bug-fix-download-missing-modules-when-envrionment-given

Install modules when using environment option

Author: Martin-Molinero

lean/commands/live/deploy.py

@@ -16,15 +16,16 @@
 import time
 from datetime import datetime
 from pathlib import Path
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, List, Optional, Tuple
 import click
 from lean.click import LeanCommand, PathParameter, ensure_options
 from lean.constants import DEFAULT_ENGINE_IMAGE
 from lean.container import container
 from lean.models.brokerages.local import all_local_brokerages, local_brokerage_data_feeds, all_local_data_feeds
 from lean.models.errors import MoreInfoError
+from lean.models.lean_config_configurer import LeanConfigConfigurer
 from lean.models.logger import Option
-from lean.models.configuration import Configuration, InfoConfiguration, InternalInputUserInput
+from lean.models.configuration import Configuration, InfoConfiguration, InternalInputUserInput, OrganzationIdConfiguration
 from lean.models.click_options import options_from_json
 from lean.models.json_module import JsonModule
 from lean.commands.live.live import live
@@ -38,12 +39,13 @@
     "real-time-handler": "QuantConnect.Lean.Engine.RealTime.LiveTradingRealTimeHandler"
 }
 
-def _raise_for_missing_properties(lean_config: Dict[str, Any], environment_name: str, lean_config_path: Path) -> None:
-    """Raises an error if any required properties are missing.
+
+def _get_configurable_modules_from_environment(lean_config: Dict[str, Any], environment_name: str) -> Tuple[LeanConfigConfigurer, List[LeanConfigConfigurer]]:
+    """Returns the configurable modules from the given environment.
 
     :param lean_config: the LEAN configuration that should be used
     :param environment_name: the name of the environment
-    :param lean_config_path: the path to the LEAN configuration file
+    :return: the configurable modules from the given environment
     """
     environment = lean_config["environments"][environment_name]
     for key in ["live-mode-brokerage", "data-queue-handler"]:
@@ -53,9 +55,34 @@ def _raise_for_missing_properties(lean_config: Dict[str, Any], environment_name:
 
     brokerage = environment["live-mode-brokerage"]
     data_queue_handlers = environment["data-queue-handler"]
-
     [brokerage_configurer] = [local_brokerage for local_brokerage in all_local_brokerages if local_brokerage.get_live_name(environment_name) == brokerage]
     data_feed_configurers = [local_data_feed for local_data_feed in all_local_data_feeds if local_data_feed.get_live_name(environment_name) in data_queue_handlers]
+    return brokerage_configurer, data_feed_configurers
+
+
+def _install_modules(modules: List[LeanConfigConfigurer], user_kwargs: Dict[str, Any]) -> None:
+    """Raises an error if any of the given modules are not installed.
+
+    :param modules: the modules to check
+    """
+    for module in modules:
+        if not module._installs:
+            continue
+        organization_id = module.get_organzation_id()
+        if organization_id is None or organization_id == "":
+            [organization_config] = module.get_config_from_type(OrganzationIdConfiguration)
+            organization_id = _get_non_interactive_organization_id(module, organization_config, user_kwargs)
+        module.ensure_module_installed(organization_id)
+
+
+def _raise_for_missing_properties(lean_config: Dict[str, Any], environment_name: str, lean_config_path: Path) -> None:
+    """Raises an error if any required properties are missing.
+
+    :param lean_config: the LEAN configuration that should be used
+    :param environment_name: the name of the environment
+    :param lean_config_path: the path to the LEAN configuration file
+    """
+    brokerage_configurer, data_feed_configurers = _get_configurable_modules_from_environment(lean_config, environment_name)
     brokerage_properties = brokerage_configurer.get_required_properties()
     data_queue_handler_properties = []
     for data_feed_configurer in data_feed_configurers:
@@ -136,7 +163,7 @@ def _configure_lean_config_interactively(lean_config: Dict[str, Any], environmen
             essential_properties_value = {config._id : config._value for config in brokerage.get_essential_configs()}
             data_feed.update_configs(essential_properties_value)
             container.logger().debug(f"interactive: essential_properties_value: {brokerage._id} {essential_properties_value}")
-            # now required properties can be fetched as per data/filter provider from esssential properties
+            # now required properties can be fetched as per data/filter provider from essential properties
             required_properties_value = {config._id : config._value for config in brokerage.get_required_configs([InternalInputUserInput])}
             data_feed.update_configs(required_properties_value)
             container.logger().debug(f"interactive: required_properties_value: {required_properties_value}")
@@ -178,6 +205,10 @@ def _get_organization_id(given_input: Optional[str], label: str) -> str:
     container.logger().debug(f"deploy._get_organization_id: user selected organization id: {organization.id}")
     return organization.id
 
+def _get_non_interactive_organization_id(module: LeanConfigConfigurer, 
+                                        organization_config: OrganzationIdConfiguration, user_kwargs: Dict[str, Any]) -> str:
+    return _get_organization_id(user_kwargs[module._convert_lean_key_to_variable(organization_config._id)], module._id)
+
 def _get_and_build_module(target_module_name: str, module_list: List[JsonModule], properties: Dict[str, Any]):
     [target_module] = [module for module in module_list if module.get_name() == target_module_name]
     # update essential properties from brokerage to datafeed
@@ -187,12 +218,12 @@ def _get_and_build_module(target_module_name: str, module_list: List[JsonModule]
     essential_properties_value = {target_module._convert_variable_to_lean_key(prop) : properties[prop] for prop in essential_properties}
     target_module.update_configs(essential_properties_value)
     container.logger().debug(f"non-interactive: essential_properties_value with module {target_module_name}: {essential_properties_value}")
-    # now required properties can be fetched as per data/filter provider from esssential properties
+    # now required properties can be fetched as per data/filter provider from essential properties
     required_properties: List[str] = []
     organization_info: Dict[str,str] = {}
     for config in target_module.get_required_configs([InternalInputUserInput]):
         if config.is_type_organization_id:
-            organization_info[config._id] = _get_organization_id(properties[target_module._convert_lean_key_to_variable(config._id)], target_module._id)
+            organization_info[config._id] = _get_non_interactive_organization_id(target_module, config, properties)
             properties[target_module._convert_lean_key_to_variable(config._id)] = organization_info[config._id]
             # skip organization id from ensure_options() because it is fetched using _get_organization_id()
             continue
@@ -362,6 +393,9 @@ def deploy(project: Path,
         raise MoreInfoError(f"The '{environment_name}' is not a live trading environment (live-mode is set to false)",
                             "https://www.lean.io/docs/lean-cli/live-trading")
 
+    env_brokerage, env_data_queue_handlers = _get_configurable_modules_from_environment(lean_config, environment_name)
+    _install_modules([env_brokerage] + env_data_queue_handlers, kwargs)
+
     _raise_for_missing_properties(lean_config, environment_name, lean_config_manager.get_lean_config_path())
 
     project_config_manager = container.project_config_manager()

lean/models/json_module.py

@@ -89,6 +89,9 @@ def get_configurations_env_values_from_name(self, target_env: str) -> List[Dict[
             env_config_values = env_config._env_and_values[target_env]
         return env_config_values
 
+    def get_config_from_type(self, config_type: Configuration) -> str:
+        return [copy.copy(config) for config in self._lean_configs if type(config) is config_type]
+
     def get_organzation_id(self) -> str:
         [organization_id] = [
             config._value for config in self._lean_configs if config.is_type_organization_id]

lean/models/lean_config_configurer.py

@@ -39,7 +39,6 @@ def _configure_environment(self, lean_config: Dict[str, Any], environment_name:
         :param lean_config: the Lean configuration dict to write to
         :param environment_name: the name of the environment to update
         """
-        self.ensure_module_installed()
         for environment_config in self.get_configurations_env_values_from_name(environment_name):
             environment_config_name = environment_config["name"]
             if self.__class__.__name__ == 'DataFeed':
@@ -86,11 +85,11 @@ def configure_credentials(self, lean_config: Dict[str, Any]) -> None:
         container.logger().debug(f"LeanConfigConfigurer.ensure_module_installed(): _save_properties for module {self._id}: {self.get_required_properties()}")
         self._save_properties(lean_config, self.get_required_properties())
 
-    def ensure_module_installed(self) -> None:
+    def ensure_module_installed(self, organization_id: str) -> None:
         if not self._is_module_installed and self._installs:
             container.logger().debug(f"LeanConfigConfigurer.ensure_module_installed(): installing module for module {self._id}: {self._product_id}")
             container.module_manager().install_module(
-                self._product_id, self.get_organzation_id())
+                self._product_id, organization_id)
             self._is_module_installed = True
 
     def _get_default(cls, lean_config: Dict[str, Any], key: str) -> Optional[Any]:

tests/commands/test_live.py

@@ -75,6 +75,12 @@ def test_live_calls_lean_runner_with_correct_algorithm_file() -> None:
     lean_runner = mock.Mock()
     container.lean_runner.override(providers.Object(lean_runner))
 
+    api_client = mock.MagicMock()
+    api_client.organizations.get_all.return_value = [
+        QCMinimalOrganization(id="abc", name="abc", type="type", ownerName="You", members=1, preferred=True)
+    ]
+    container.api_client.override(providers.Object(api_client))
+
     result = CliRunner().invoke(lean, ["live", "Python Project", "--environment", "live-paper"])
 
     traceback.print_exception(*result.exc_info)
@@ -134,6 +140,12 @@ def test_live_calls_lean_runner_with_default_output_directory() -> None:
     lean_runner = mock.Mock()
     container.lean_runner.override(providers.Object(lean_runner))
 
+    api_client = mock.MagicMock()
+    api_client.organizations.get_all.return_value = [
+        QCMinimalOrganization(id="abc", name="abc", type="type", ownerName="You", members=1, preferred=True)
+    ]
+    container.api_client.override(providers.Object(api_client))
+
     result = CliRunner().invoke(lean, ["live", "Python Project", "--environment", "live-paper"])
 
     assert result.exit_code == 0
@@ -155,6 +167,12 @@ def test_live_calls_lean_runner_with_custom_output_directory() -> None:
     lean_runner = mock.Mock()
     container.lean_runner.override(providers.Object(lean_runner))
 
+    api_client = mock.MagicMock()
+    api_client.organizations.get_all.return_value = [
+        QCMinimalOrganization(id="abc", name="abc", type="type", ownerName="You", members=1, preferred=True)
+    ]
+    container.api_client.override(providers.Object(api_client))
+    
     result = CliRunner().invoke(lean, ["live",
                                        "Python Project",
                                        "--environment", "live-paper",
@@ -179,6 +197,12 @@ def test_live_calls_lean_runner_with_release_mode() -> None:
     lean_runner = mock.Mock()
     container.lean_runner.override(providers.Object(lean_runner))
 
+    api_client = mock.MagicMock()
+    api_client.organizations.get_all.return_value = [
+        QCMinimalOrganization(id="abc", name="abc", type="type", ownerName="You", members=1, preferred=True)
+    ]
+    container.api_client.override(providers.Object(api_client))
+
     result = CliRunner().invoke(lean, ["live", "CSharp Project", "--environment", "live-paper", "--release"])
 
     assert result.exit_code == 0
@@ -203,6 +227,12 @@ def test_live_calls_lean_runner_with_detach() -> None:
     lean_runner = mock.Mock()
     container.lean_runner.override(providers.Object(lean_runner))
 
+    api_client = mock.MagicMock()
+    api_client.organizations.get_all.return_value = [
+        QCMinimalOrganization(id="abc", name="abc", type="type", ownerName="You", members=1, preferred=True)
+    ]
+    container.api_client.override(providers.Object(api_client))
+
     result = CliRunner().invoke(lean, ["live", "Python Project", "--environment", "live-paper", "--detach"])
 
     assert result.exit_code == 0