Support extra docker configs (#360)

* Get additional docker configs in local deployment commands.

This accepts only device_requests for now.

* Update new docker extra config option documentation

* Support mounting volumes using extra docker configs

* Update readme

* Update readme

Author: jhonabreul

README.md

@@ -160,6 +160,8 @@ Options:
   --python-venv TEXT              The path of the python virtual environment to be used
   --update                        Pull the LEAN engine image before running the backtest
   --backtest-name TEXT            Backtest name
+  --extra-docker-config TEXT      Extra docker configuration as a JSON string. For more information https://docker-
+                                  py.readthedocs.io/en/stable/containers.html
   --no-update                     Use the local LEAN engine image instead of pulling the latest version
   --lean-config FILE              The Lean configuration file that should be used (defaults to the nearest lean.json)
   --verbose                       Enable debug logging
@@ -1098,6 +1100,8 @@ Options:
                                   holdings
   --update                        Pull the LEAN engine image before starting live trading
   --show-secrets                  Show secrets as they are input
+  --extra-docker-config TEXT      Extra docker configuration as a JSON string. For more information https://docker-
+                                  py.readthedocs.io/en/stable/containers.html
   --no-update                     Use the local LEAN engine image instead of pulling the latest version
   --lean-config FILE              The Lean configuration file that should be used (defaults to the nearest lean.json)
   --verbose                       Enable debug logging
@@ -1383,6 +1387,8 @@ Options:
   --estimate                      Estimate optimization runtime without running it
   --max-concurrent-backtests INTEGER RANGE
                                   Maximum number of concurrent backtests to run  [x>=1]
+  --extra-docker-config TEXT      Extra docker configuration as a JSON string. For more information https://docker-
+                                  py.readthedocs.io/en/stable/containers.html
   --no-update                     Use the local LEAN engine image instead of pulling the latest version
   --lean-config FILE              The Lean configuration file that should be used (defaults to the nearest lean.json)
   --verbose                       Enable debug logging
@@ -1505,6 +1511,8 @@ Options:
   --no-open                       Don't open the Jupyter Lab environment in the browser after starting it
   --image TEXT                    The LEAN research image to use (defaults to quantconnect/research:latest)
   --update                        Pull the LEAN research image before starting the research environment
+  --extra-docker-config TEXT      Extra docker configuration as a JSON string. For more information https://docker-
+                                  py.readthedocs.io/en/stable/containers.html
   --no-update                     Use the local LEAN research image instead of pulling the latest version
   --lean-config FILE              The Lean configuration file that should be used (defaults to the nearest lean.json)
   --verbose                       Enable debug logging

lean/commands/backtest.py

@@ -11,7 +11,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-
 from pathlib import Path
 from typing import List, Optional, Tuple
 from click import command, option, argument, Choice
@@ -289,6 +288,11 @@ def _select_organization() -> QCMinimalOrganization:
               type=(str, str),
               multiple=True,
               hidden=True)
+@option("--extra-docker-config",
+              type=str,
+              default="{}",
+              help="Extra docker configuration as a JSON string. "
+                   "For more information https://docker-py.readthedocs.io/en/stable/containers.html")
 @option("--no-update",
               is_flag=True,
               default=False,
@@ -307,6 +311,7 @@ def backtest(project: Path,
              backtest_name: str,
              addon_module: Optional[List[str]],
              extra_config: Optional[Tuple[str, str]],
+             extra_docker_config: Optional[str],
              no_update: bool,
              **kwargs) -> None:
     """Backtest a project locally using Docker.
@@ -324,6 +329,8 @@ def backtest(project: Path,
     Alternatively you can set the default engine image for all commands using `lean config set engine-image <image>`.
     """
     from datetime import datetime
+    from json import loads
+
     logger = container.logger
     project_manager = container.project_manager
     algorithm_file = project_manager.find_algorithm_file(Path(project))
@@ -407,4 +414,5 @@ def backtest(project: Path,
                          engine_image,
                          debugging_method,
                          release,
-                         detach)
+                         detach,
+                         loads(extra_docker_config))

lean/commands/live/deploy.py

@@ -255,6 +255,11 @@ def _get_default_value(key: str) -> Optional[Any]:
               type=(str, str),
               multiple=True,
               hidden=True)
+@option("--extra-docker-config",
+              type=str,
+              default="{}",
+              help="Extra docker configuration as a JSON string. "
+                   "For more information https://docker-py.readthedocs.io/en/stable/containers.html")
 @option("--no-update",
               is_flag=True,
               default=False,
@@ -275,6 +280,7 @@ def deploy(project: Path,
            show_secrets: bool,
            addon_module: Optional[List[str]],
            extra_config: Optional[Tuple[str, str]],
+           extra_docker_config: Optional[str],
            no_update: bool,
            **kwargs) -> None:
     """Start live trading a project locally using Docker.
@@ -299,6 +305,7 @@ def deploy(project: Path,
     """
     from copy import copy
     from datetime import datetime
+    from json import loads
     # Reset globals so we reload everything in between tests
     global _cached_lean_config
     _cached_lean_config = None
@@ -430,4 +437,4 @@ def deploy(project: Path,
             raise RuntimeError(f"InteractiveBrokers is currently not supported for ARM hosts")
 
     lean_runner = container.lean_runner
-    lean_runner.run_lean(lean_config, environment_name, algorithm_file, output, engine_image, None, release, detach)
+    lean_runner.run_lean(lean_config, environment_name, algorithm_file, output, engine_image, None, release, detach, loads(extra_docker_config))

lean/commands/optimize.py

@@ -18,6 +18,7 @@
 from click import command, argument, option, Choice, IntRange
 
 from lean.click import LeanCommand, PathParameter, ensure_options
+from lean.components.docker.lean_runner import LeanRunner
 from lean.constants import DEFAULT_ENGINE_IMAGE
 from lean.container import container
 from lean.models.api import QCParameter, QCBacktest
@@ -119,6 +120,11 @@ def get_filename_timestamp(path: Path) -> datetime:
               type=(str, str),
               multiple=True,
               hidden=True)
+@option("--extra-docker-config",
+              type=str,
+              default="{}",
+              help="Extra docker configuration as a JSON string. "
+                   "For more information https://docker-py.readthedocs.io/en/stable/containers.html")
 @option("--no-update",
               is_flag=True,
               default=False,
@@ -139,6 +145,7 @@ def optimize(project: Path,
              max_concurrent_backtests: Optional[int],
              addon_module: Optional[List[str]],
              extra_config: Optional[Tuple[str, str]],
+             extra_docker_config: Optional[str],
              no_update: bool) -> None:
     """Optimize a project's parameters locally using Docker.
 
@@ -308,6 +315,9 @@ def optimize(project: Path,
     )
     container.update_manager.pull_docker_image_if_necessary(engine_image, update, no_update)
 
+    # Add known additional run options from the extra docker config
+    LeanRunner.parse_extra_docker_config(run_options, loads(extra_docker_config))
+
     project_manager.copy_code(algorithm_file.parent, output / "code")
 
     success = container.docker_manager.run_image(engine_image, **run_options)

lean/commands/research.py

@@ -15,6 +15,7 @@
 from typing import Optional, Tuple
 from click import command, argument, option, Choice
 from lean.click import LeanCommand, PathParameter
+from lean.components.docker.lean_runner import LeanRunner
 from lean.constants import DEFAULT_RESEARCH_IMAGE, LEAN_ROOT_PATH
 from lean.container import container
 from lean.models.data_providers import QuantConnectDataProvider, all_data_providers
@@ -65,6 +66,11 @@ def _check_docker_output(chunk: str, port: int) -> None:
               type=(str, str),
               multiple=True,
               hidden=True)
+@option("--extra-docker-config",
+              type=str,
+              default="{}",
+              help="Extra docker configuration as a JSON string. "
+                   "For more information https://docker-py.readthedocs.io/en/stable/containers.html")
 @option("--no-update",
               is_flag=True,
               default=False,
@@ -79,6 +85,7 @@ def research(project: Path,
              image: Optional[str],
              update: bool,
              extra_config: Optional[Tuple[str, str]],
+             extra_docker_config: Optional[str],
              no_update: bool,
              **kwargs) -> None:
     """Run a Jupyter Lab environment locally using Docker.
@@ -89,6 +96,7 @@ def research(project: Path,
     """
     from docker.types import Mount
     from docker.errors import APIError
+    from json import loads
 
     logger = container.logger
 
@@ -116,7 +124,7 @@ def research(project: Path,
     # Set extra config
     for key, value in extra_config:
         lean_config[key] = value
-        
+
     run_options = lean_runner.get_basic_docker_config(lean_config,
                                                       algorithm_file,
                                                       temp_manager.create_temporary_directory(),
@@ -160,6 +168,9 @@ def research(project: Path,
     # Run the script that starts Jupyter Lab when all set up has been done
     run_options["commands"].append("./start.sh")
 
+    # Add known additional run options from the extra docker config
+    LeanRunner.parse_extra_docker_config(run_options, loads(extra_docker_config))
+
     project_config_manager = container.project_config_manager
     cli_config_manager = container.cli_config_manager
 

lean/components/docker/lean_runner.py

@@ -13,6 +13,7 @@
 
 from pathlib import Path
 from typing import Any, Dict, Optional, List
+
 from lean.components.cloud.module_manager import ModuleManager
 from lean.components.config.lean_config_manager import LeanConfigManager
 from lean.components.config.output_config_manager import OutputConfigManager
@@ -70,7 +71,8 @@ def run_lean(self,
                  image: DockerImage,
                  debugging_method: Optional[DebuggingMethod],
                  release: bool,
-                 detach: bool) -> None:
+                 detach: bool,
+                 extra_docker_config: Optional[Dict[str, Any]] = None) -> None:
         """Runs the LEAN engine locally in Docker.
 
         Raises an error if something goes wrong.
@@ -83,6 +85,7 @@ def run_lean(self,
         :param debugging_method: the debugging method if debugging needs to be enabled, None if not
         :param release: whether C# projects should be compiled in release configuration instead of debug
         :param detach: whether LEAN should run in a detached container
+        :param extra_docker_config: additional docker configurations
         """
         project_dir = algorithm_file.parent
 
@@ -95,6 +98,9 @@ def run_lean(self,
                                                    release,
                                                    detach)
 
+        # Add known additional run options from the extra docker config
+        self.parse_extra_docker_config(run_options, extra_docker_config)
+
         # Set up PTVSD debugging
         if debugging_method == DebuggingMethod.PTVSD:
             run_options["ports"]["5678"] = "5678"
@@ -762,3 +768,19 @@ def mount_project_and_library_directories(self, project_dir: Path, run_options:
                 "bind": "/Library",
                 "mode": "rw"
             }
+
+    @staticmethod
+    def parse_extra_docker_config(run_options: Dict[str, Any], extra_docker_config: Optional[Dict[str, Any]]) -> None:
+        from docker.types import DeviceRequest
+        # Add known additional run options from the extra docker config.
+        # For now, only device_requests is supported
+        if extra_docker_config is not None:
+            if "device_requests" in extra_docker_config:
+                run_options["device_requests"] = [DeviceRequest(**device_request)
+                                                  for device_request in extra_docker_config["device_requests"]]
+
+            if "volumes" in extra_docker_config:
+                volumes = run_options.get("volumes")
+                if not volumes:
+                    volumes = run_options["volumes"] = {}
+                volumes.update(extra_docker_config["volumes"])

tests/commands/test_backtest.py

@@ -57,7 +57,8 @@ def test_backtest_calls_lean_runner_with_correct_algorithm_file() -> None:
                                                  ENGINE_IMAGE,
                                                  None,
                                                  False,
-                                                 False)
+                                                 False,
+                                                 {})
 
 
 def test_backtest_calls_lean_runner_with_default_output_directory() -> None:
@@ -88,7 +89,8 @@ def test_backtest_calls_lean_runner_with_custom_output_directory() -> None:
                                                  ENGINE_IMAGE,
                                                  None,
                                                  False,
-                                                 False)
+                                                 False,
+                                                 {})
 
 
 def test_backtest_calls_lean_runner_with_release_mode() -> None:
@@ -105,7 +107,8 @@ def test_backtest_calls_lean_runner_with_release_mode() -> None:
                                                  ENGINE_IMAGE,
                                                  None,
                                                  True,
-                                                 False)
+                                                 False,
+                                                 {})
 
 
 def test_backtest_calls_lean_runner_with_detach() -> None:
@@ -122,7 +125,8 @@ def test_backtest_calls_lean_runner_with_detach() -> None:
                                                  ENGINE_IMAGE,
                                                  None,
                                                  False,
-                                                 True)
+                                                 True,
+                                                 {})
 
 
 def test_backtest_aborts_when_project_does_not_exist() -> None:
@@ -163,7 +167,8 @@ def test_backtest_forces_update_when_update_option_given() -> None:
                                                  ENGINE_IMAGE,
                                                  None,
                                                  False,
-                                                 False)
+                                                 False,
+                                                 {})
 
 
 def test_backtest_passes_custom_image_to_lean_runner_when_set_in_config() -> None:
@@ -182,7 +187,8 @@ def test_backtest_passes_custom_image_to_lean_runner_when_set_in_config() -> Non
                                                  DockerImage(name="custom/lean", tag="123"),
                                                  None,
                                                  False,
-                                                 False)
+                                                 False,
+                                                 {})
 
 
 def test_backtest_passes_custom_image_to_lean_runner_when_given_as_option() -> None:
@@ -201,7 +207,8 @@ def test_backtest_passes_custom_image_to_lean_runner_when_given_as_option() -> N
                                                  DockerImage(name="custom/lean", tag="456"),
                                                  None,
                                                  False,
-                                                 False)
+                                                 False,
+                                                 {})
 
 
 @pytest.mark.parametrize("python_venv", ["Custom-venv",
@@ -289,7 +296,8 @@ def test_backtest_passes_correct_debugging_method_to_lean_runner(value: str, deb
                                                  ENGINE_IMAGE,
                                                  debugging_method,
                                                  False,
-                                                 False)
+                                                 False,
+                                                 {})
 
 
 def test_backtest_auto_updates_outdated_python_pycharm_debug_config() -> None:
@@ -649,3 +657,31 @@ def test_backtest_adds_python_libraries_path_to_lean_config() -> None:
     expected_library_path = (Path("/") / library_path.relative_to(lean_cli_root_dir)).as_posix()
 
     assert expected_library_path in lean_config.get('python-additional-paths')
+
+
+def test_backtest_calls_lean_runner_with_extra_docker_config() -> None:
+    create_fake_lean_cli_directory()
+
+    result = CliRunner().invoke(lean, ["backtest", "Python Project",
+                                       "--extra-docker-config",
+                                       '{"device_requests": [{"count": -1, "capabilities": [["compute"]]}],'
+                                       '"volumes": {"extra/path": {"bind": "/extra/path", "mode": "rw"}}}'])
+
+    assert result.exit_code == 0
+
+    container.lean_runner.run_lean.assert_called_once_with(mock.ANY,
+                                                           "backtesting",
+                                                           Path("Python Project/main.py").resolve(),
+                                                           mock.ANY,
+                                                           ENGINE_IMAGE,
+                                                           None,
+                                                           False,
+                                                           False,
+                                                           {
+                                                               "device_requests": [
+                                                                   {"count": -1, "capabilities": [["compute"]]}
+                                                               ],
+                                                               "volumes": {
+                                                                   "extra/path": {"bind": "/extra/path", "mode": "rw"}
+                                                               }
+                                                           })