feat(gitlab): add pagination params for repository tree and issues

llama-index-integrations/readers/llama-index-readers-gitlab/llama_index/readers/gitlab/__init__.py

@@ -1,7 +1,6 @@
 from llama_index.readers.gitlab.issues.base import GitLabIssuesReader
 from llama_index.readers.gitlab.repository.base import GitLabRepositoryReader
 
-
 __all__ = [
     "GitLabIssuesReader",
     "GitLabRepositoryReader",

llama-index-integrations/readers/llama-index-readers-gitlab/llama_index/readers/gitlab/issues/base.py

@@ -1,14 +1,15 @@
 """GitLab issues reader."""
 
-from datetime import datetime
 import enum
+from datetime import datetime
 from typing import Any, List, Optional, Union
 
-import gitlab
-from gitlab.v4.objects import Issue as GitLabIssue
 from llama_index.core import Document
 from llama_index.core.readers.base import BaseReader
 
+import gitlab
+from gitlab.v4.objects import Issue as GitLabIssue
+
 
 class GitLabIssuesReader(BaseReader):
     """
@@ -133,6 +134,7 @@ def load_data(
         state: Optional[IssueState] = IssueState.OPEN,
         updated_after: Optional[datetime] = None,
         updated_before: Optional[datetime] = None,
+        get_all: bool = False,
         **kwargs: Any,
     ) -> List[Document]:
         """
@@ -167,6 +169,7 @@ def load_data(
             - state: State of the issues to retrieve.
             - updated_after: Filter issues updated after the specified date.
             - updated_before: Filter issues updated before the specified date.
+            - get_all: Get all the items without pagination (for a long lists).
 
 
         Returns:
@@ -188,6 +191,7 @@ def load_data(
             "state": state.value if state else None,
             "updated_after": to_gitlab_datetime_format(updated_after),
             "updated_before": to_gitlab_datetime_format(updated_before),
+            "get_all": get_all,
         }
 
         if isinstance(assignee, str):

llama-index-integrations/readers/llama-index-readers-gitlab/llama_index/readers/gitlab/repository/base.py

@@ -2,10 +2,11 @@
 
 from typing import List, Optional
 
-import gitlab
 from llama_index.core import Document
 from llama_index.core.readers.base import BaseReader
 
+import gitlab
+
 
 class GitLabRepositoryReader(BaseReader):
     def __init__(
@@ -52,6 +53,7 @@ def load_data(
         file_path: Optional[str] = None,
         path: Optional[str] = None,
         recursive: bool = False,
+        iterator: bool = False,
     ) -> List[Document]:
         """
         Load data from a GitLab repository.
@@ -61,6 +63,7 @@ def load_data(
             file_path: Path to the file to load.
             path: Path to the directory to load.
             recursive: Whether to load files recursively.
+            iterator: Return a generator handling API pagination automatically
 
         Returns:
             List[Document]: List of documents loaded from the repository
@@ -75,6 +78,7 @@ def load_data(
             "ref": ref,
             "path": path,
             "recursive": recursive,
+            "iterator": iterator,
         }
 
         filtered_params = {k: v for k, v in params.items() if v is not None}

llama-index-integrations/readers/llama-index-readers-gitlab/pyproject.toml

@@ -26,7 +26,7 @@ dev = [
 
 [project]
 name = "llama-index-readers-gitlab"
-version = "0.4.1"
+version = "0.5.1"
 description = "llama-index readers gitlab integration"
 authors = [{name = "Jiacheng Zhang", email = "[email protected]"}]
 requires-python = ">=3.9,<4.0"

llama-index-integrations/readers/llama-index-readers-gitlab/tests/test_readers_gitlab_issues.py

@@ -1,10 +1,11 @@
-import pytest
 from unittest.mock import MagicMock
-from llama_index.readers.gitlab import GitLabIssuesReader
+
 import gitlab
+import pytest
 
 # import so that pants sees it as a dependency
 import pytest_mock  # noqa
+from llama_index.readers.gitlab import GitLabIssuesReader
 
 
 @pytest.fixture()

llama-index-integrations/readers/llama-index-readers-gitlab/tests/test_readers_gitlab_repository.py

@@ -1,7 +1,8 @@
-import pytest
 from unittest import mock
-from llama_index.readers.gitlab import GitLabRepositoryReader
+
+import pytest
 from llama_index.core import Document
+from llama_index.readers.gitlab import GitLabRepositoryReader
 
 
 @pytest.fixture()