Merge pull request #353 from QuantConnect/bug-352-duplicate-project-name-handling

Lean cloud pull improvements

Author: Martin-Molinero

lean/components/cloud/pull_manager.py

@@ -22,6 +22,8 @@
 from lean.models.api import QCProject, QCLanguage, QCProjectLibrary
 from lean.models.errors import RequestFailedError
 from lean.models.utils import LeanLibraryReference
+from lean.components.config.storage import safe_save
+
 
 class PullManager:
     """The PullManager class is responsible for synchronizing cloud projects to the local drive."""
@@ -153,7 +155,8 @@ def _pull_project(self, project: QCProject) -> Path:
         :param project: the cloud project to pull
         :return the actual local path of the project
         """
-        local_project_path = self._project_manager.get_local_project_path(project.name, project.projectId)
+        local_project_path = self._project_manager.get_local_project_path(project.name, project.projectId,
+                                                                          allow_corrupted=True)
         local_project_name = local_project_path.relative_to(Path.cwd()).as_posix()
         # Check if cloud project has invalid name, if so update it and inform user.
         if local_project_name != project.name:
@@ -215,14 +218,12 @@ def _pull_files(self, project: QCProject, local_project_path: Path) -> None:
 
             local_file_path.parent.mkdir(parents=True, exist_ok=True)
 
-            with local_file_path.open("w+", encoding="utf-8") as local_file:
-                if cloud_file.content != "":
-                    # Make sure we always work with unix line endings in memory,
-                    # so they can be properly translated to the local OS line endings when writing to disk.
-                    content = cloud_file.content.replace("\r\n", "\n")
-                    if not content.endswith("\n"):
-                        content += "\n"
-                    local_file.write(content)
+            # Make sure we always work with unix line endings in memory,
+            # so they can be properly translated to the local OS line endings when writing to disk.
+            content = cloud_file.content.replace("\r\n", "\n")
+            if content != "" and not content.endswith("\n"):
+                content += "\n"
+            safe_save(content, local_file_path)
 
             self._project_manager.update_last_modified_time(local_file_path, cloud_file.modified)
             self._logger.info(f"Successfully pulled '{project.name}/{cloud_file.name}'")

lean/components/config/storage.py

@@ -75,6 +75,13 @@ def __init__(self, file: str) -> None:
         else:
             self._data = {}
 
+    def is_empty(self) -> bool:
+        """Determines if this storage file is empty
+
+        :return: True if this storage file is empty
+        """
+        return len(self._data) == 0
+
     def get(self, key: str, default: Any = None) -> Any:
         """Returns the value assigned to the given key.
 

lean/components/util/project_manager.py

@@ -203,16 +203,18 @@ def delete_project(self, project_dir: Path) -> None:
             raise RuntimeError(f"Failed to delete project. Could not find the specified path {project_dir}.")
 
 
-    def get_local_project_path(self, project_name: str, cloud_id: Optional[int] = None, local_id: Optional[int] = None) -> Path:
+    def get_local_project_path(self, project_name: str, cloud_id: Optional[int] = None, local_id: Optional[int] = None,
+                               allow_corrupted: Optional[bool] = False) -> Path:
         """Returns the local path where a certain cloud project should be stored.
 
         If two cloud projects are named "Project", they are pulled to ./Project and ./Project 2.
 
         If you push a project with unsupported cloud name, a supported project name would be assigned.
 
-        :param project_name: the cloud project to get the project path of
-        :param cloud_id: the cloud project to get the project path of
-        :param local_id: the cloud project to get the project path of
+        :param project_name: the cloud project name to get the project path of
+        :param cloud_id: the cloud project id to get the project path of
+        :param local_id: the cloud project local id to get the project path of
+        :param allow_corrupted: true if a corrupted path can be used
         :return: the path to the local project directory
         """
 
@@ -226,6 +228,7 @@ def get_local_project_path(self, project_name: str, cloud_id: Optional[int] = No
 
         current_index = 1
         while True:
+            # we first check the current project name
             path_suffix = "" if current_index == 1 else f" {current_index}"
             current_path = Path.cwd() / (local_path + path_suffix)
 
@@ -234,14 +237,33 @@ def get_local_project_path(self, project_name: str, cloud_id: Optional[int] = No
 
             if cloud_id is not None:
                 current_project_config = self._project_config_manager.get_project_config(current_path)
-                if current_project_config.get("cloud-id") == cloud_id:
+                if current_project_config.is_empty():
+                    self._logger.error(f"'{current_path}' '{PROJECT_CONFIG_FILE_NAME}' file is corrupted!")
+                    if allow_corrupted:
+                        return current_path
+                elif current_project_config.get("cloud-id") == cloud_id:
                     return current_path
 
             if local_id is not None:
                 current_project_config = self._project_config_manager.get_project_config(current_path)
-                if current_project_config.get("local-id") == local_id:
+                if current_project_config.is_empty():
+                    self._logger.error(f"'{current_path}' '{PROJECT_CONFIG_FILE_NAME}' file is corrupted!")
+                    if allow_corrupted:
+                        return current_path
+                elif current_project_config.get("local-id") == local_id:
                     return current_path
 
+            if current_index == 1:
+                from re import findall
+
+                ints_in_name = findall(r"(\s\d+)$", local_path)
+                if len(ints_in_name) != 0:
+                    # the current project name already exists, and it has a 'space + int' at the end of it
+                    # so, we take that int and increment it
+                    int_value = ints_in_name[0]
+                    # we remove the int from the name because we will re add it in the loop
+                    local_path = local_path[:-len(int_value)]
+                    current_index = int(int_value)
             current_index += 1
 
     def rename_project_and_contents(self, old_path: Path, new_path: Path,) -> None:

tests/components/cloud/test_cloud_project_manager.py

@@ -36,6 +36,7 @@ def test_get_cloud_project_pushing_new_project():
 
     project_config = mock.Mock()
     project_config.get = mock.MagicMock(return_value=cloud_project.projectId)
+    project_config.is_empty = mock.MagicMock(return_value=False)
     project_config_manager = mock.Mock()
     project_config_manager.try_get_project_config = mock.MagicMock(return_value=None)
     project_config_manager.get_project_config = mock.MagicMock(return_value=project_config)