Merge pull request #214 from aboutcode-org/add-settings

Add settings

Author: pombredanne

CHANGELOG.rst

@@ -1,6 +1,42 @@
 Changelog
 =========
 
+v0.13.2
+-----------
+
+- Speed up downloads with asyncio
+
+- New settings featuring environment variables and .env file to store settings and defaults.
+
+  - This also changes the CACHE_THIRDPARTY_DIR environment variable: it used to default first
+    to ".cache/python_inspector" and if not writable, it would fallback to home
+    "~/.cache/python_inspector".  The new behavior is to only use the "~/.cache/python_inspector"
+    in the home directory. You can configure this directory to any other value.
+
+  - Another change is that pypi.org is no longer systematically added as an index URL for
+    resolution. Instead the list of configured index URLs is used, and only defaults to pypi.org
+    if not provided.
+
+  - Another change is that it is possible to only use the provided or configured index URLs
+    and skip other URLs from requirements that are not in these configured URLs.
+
+  - Calling utils_pypi.download_sdist or utils_pypi.download_wheel requires a non-empty list
+    of PypiSimpleRepository.
+
+  - python_inspector.utils_pypi.Distribution.download_url is now a method, not a property
+
+  - The command line has again a default OS and Python version set.
+
+  - Default option values are reported in the JSON results. They were skipped before.
+
+- Drop support for running on Python 3.8. You can still resolve dependencies for Python 3.8.
+  The default command line tool Python version used for resolution is now 3.9.
+
+- Add support for the latest Python and OS versions.
+
+- Merge latest skeleton and adopt ruff for code formatting.
+
+
 v0.13.1
 -----------
 

requirements.txt

@@ -14,6 +14,8 @@ packaging==24.2
 packvers==21.5
 pip-requirements-parser==32.0.1
 pkginfo2==30.0.0
+pydantic_settings == 2.8.1
+pydantic == 2.11.2
 pyparsing==3.0.9
 PyYAML==6.0
 requests==2.28.1

setup.cfg

@@ -71,6 +71,8 @@ install_requires =
     packvers >= 21.5
     aiohttp >= 3.8
     aiofiles >= 23.1
+    pydantic >= 2.10.0
+    pydantic_settings >= 2.8.0
 
 [options.packages.find]
 where = src

src/python_inspector/__init__.py

@@ -7,4 +7,9 @@
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
 
-DEFAULT_PYTHON_VERSION = "3.8"
+from python_inspector import settings
+
+# Initialize global settings
+pyinspector_settings = settings.Settings()
+
+settings.create_cache_directory(pyinspector_settings.CACHE_THIRDPARTY_DIR)

src/python_inspector/api.py

@@ -29,6 +29,7 @@
 from _packagedcode.pypi import can_process_dependent_package
 from _packagedcode.pypi import get_resolved_purl
 from python_inspector import dependencies
+from python_inspector import pyinspector_settings as settings
 from python_inspector import utils
 from python_inspector import utils_pypi
 from python_inspector.package_data import get_pypi_data_from_purl
@@ -42,8 +43,8 @@
 from python_inspector.resolution import get_requirements_from_python_manifest
 from python_inspector.utils import Candidate
 from python_inspector.utils_pypi import PLATFORMS_BY_OS
-from python_inspector.utils_pypi import PYPI_SIMPLE_URL
 from python_inspector.utils_pypi import Environment
+from python_inspector.utils_pypi import PypiSimpleRepository
 from python_inspector.utils_pypi import valid_python_versions
 
 
@@ -80,7 +81,7 @@ def resolve_dependencies(
     specifiers=tuple(),
     python_version=None,
     operating_system=None,
-    index_urls=tuple([PYPI_SIMPLE_URL]),
+    index_urls: tuple[str, ...] = settings.INDEX_URL,
     pdt_output=None,
     netrc_file=None,
     max_rounds=200000,
@@ -103,7 +104,7 @@ def resolve_dependencies(
     linux OS.
 
     Download from the provided PyPI simple index_urls INDEX(s) URLs defaulting
-    to PyPI.org
+    to PyPI.org or a configured setting.
     """
 
     if not operating_system:
@@ -148,9 +149,6 @@ def resolve_dependencies(
 
     files = []
 
-    if PYPI_SIMPLE_URL not in index_urls:
-        index_urls = tuple([PYPI_SIMPLE_URL]) + tuple(index_urls)
-
     # requirements
     for req_file in requirement_files:
         deps = dependencies.get_dependencies_from_requirements(requirements_file=req_file)
@@ -249,29 +247,32 @@ def resolve_dependencies(
     if verbose:
         printer(f"environment: {environment}")
 
-    repos = []
+    repos_by_url = {}
     if not use_pypi_json_api:
         # Collect PyPI repos
+        use_only_confed = settings.USE_ONLY_CONFIGURED_INDEX_URLS
         for index_url in index_urls:
             index_url = index_url.strip("/")
-            existing = utils_pypi.DEFAULT_PYPI_REPOS_BY_URL.get(index_url)
-            if existing:
-                existing.use_cached_index = use_cached_index
-                repos.append(existing)
-            else:
-                credentials = None
-                if parsed_netrc:
-                    login, password = utils.get_netrc_auth(index_url, parsed_netrc)
-                    credentials = (
-                        dict(login=login, password=password) if login and password else None
-                    )
-                repo = utils_pypi.PypiSimpleRepository(
-                    index_url=index_url,
-                    use_cached_index=use_cached_index,
-                    credentials=credentials,
-                )
-                repos.append(repo)
+            if use_only_confed and index_url not in settings.INDEX_URL:
+                if verbose:
+                    printer(f"Skipping index URL unknown in settings: {index_url!r}")
+                continue
+            if index_url in repos_by_url:
+                continue
+
+            credentials = None
+            if parsed_netrc:
+                login, password = utils.get_netrc_auth(index_url, parsed_netrc)
+                if login and password:
+                    credentials = dict(login=login, password=password)
+            repo = utils_pypi.PypiSimpleRepository(
+                index_url=index_url,
+                use_cached_index=use_cached_index,
+                credentials=credentials,
+            )
+            repos_by_url[index_url] = repo
 
+    repos = repos_by_url.values()
     if verbose:
         printer("repos:")
         for repo in repos:

src/python_inspector/package_data.py

@@ -168,7 +168,7 @@ async def get_wheel_download_urls(
             environment=environment,
             python_version=python_version,
         ):
-            download_urls.append(await wheel.download_url)
+            download_urls.append(await wheel.download_url(repo))
     return download_urls
 
 
@@ -186,4 +186,4 @@ async def get_sdist_download_url(
             python_version=python_version,
         )
         if sdist:
-            return await sdist.download_url
+            return await sdist.download_url(repo)

src/python_inspector/resolution.py

@@ -39,6 +39,7 @@
 from _packagedcode.pypi import PythonSetupPyHandler
 from _packagedcode.pypi import SetupCfgHandler
 from _packagedcode.pypi import can_process_dependent_package
+from python_inspector import pyinspector_settings as settings
 from python_inspector import utils_pypi
 from python_inspector.error import NoVersionsFound
 from python_inspector.setup_py_live_eval import iter_requirements
@@ -107,7 +108,7 @@ def get_deps_from_distribution(
     deps = []
     for package_data in handler.parse(location):
         dependencies = package_data.dependencies
-        deps.extend(dependencies)
+        deps.extend(dependencies=dependencies)
     return deps
 
 
@@ -211,21 +212,21 @@ async def fetch_and_extract_sdist(
 def get_sdist_file_path_from_filename(sdist):
     if sdist.endswith(".tar.gz"):
         sdist_file = sdist.rstrip(".tar.gz")
-        with tarfile.open(os.path.join(utils_pypi.CACHE_THIRDPARTY_DIR, sdist)) as file:
+        with tarfile.open(os.path.join(settings.CACHE_THIRDPARTY_DIR, sdist)) as file:
             file.extractall(
-                os.path.join(utils_pypi.CACHE_THIRDPARTY_DIR, "extracted_sdists", sdist_file)
+                os.path.join(settings.CACHE_THIRDPARTY_DIR, "extracted_sdists", sdist_file)
             )
     elif sdist.endswith(".zip"):
         sdist_file = sdist.rstrip(".zip")
-        with ZipFile(os.path.join(utils_pypi.CACHE_THIRDPARTY_DIR, sdist)) as zip:
+        with ZipFile(os.path.join(settings.CACHE_THIRDPARTY_DIR, sdist)) as zip:
             zip.extractall(
-                os.path.join(utils_pypi.CACHE_THIRDPARTY_DIR, "extracted_sdists", sdist_file)
+                os.path.join(settings.CACHE_THIRDPARTY_DIR, "extracted_sdists", sdist_file)
             )
 
     else:
         raise Exception(f"Unable to extract sdist {sdist}")
 
-    return os.path.join(utils_pypi.CACHE_THIRDPARTY_DIR, "extracted_sdists", sdist_file, sdist_file)
+    return os.path.join(settings.CACHE_THIRDPARTY_DIR, "extracted_sdists", sdist_file, sdist_file)
 
 
 def get_requirements_from_dependencies(
@@ -444,7 +445,7 @@ async def _get_versions_for_package_from_pypi_json_api(self, name: str) -> List[
         api_url = f"https://pypi.org/pypi/{name}/json"
         resp = await get_response_async(api_url)
         if not resp:
-            self.versions_by_package[name] = []
+            return []
         releases = resp.get("releases") or {}
         return releases.keys() or []
 
@@ -497,7 +498,7 @@ async def _get_requirements_for_package_from_pypi_simple(
 
         if wheels:
             for wheel in wheels:
-                wheel_location = os.path.join(utils_pypi.CACHE_THIRDPARTY_DIR, wheel)
+                wheel_location = os.path.join(settings.CACHE_THIRDPARTY_DIR, wheel)
                 requirements = get_requirements_from_distribution(
                     handler=PypiWheelHandler,
                     location=wheel_location,
@@ -596,7 +597,8 @@ def _iter_matches(
         name = remove_extras(identifier=identifier)
         bad_versions = {c.version for c in incompatibilities[identifier]}
         extras = {e for r in requirements[identifier] for e in r.extras}
-        versions = self.get_versions_for_package(name)
+        versions = []
+        versions.extend(self.get_versions_for_package(name=name))
 
         if not versions:
             if self.ignore_errors:

src/python_inspector/resolve_cli.py

@@ -13,6 +13,7 @@
 
 import click
 
+from python_inspector import pyinspector_settings as settings
 from python_inspector import utils_pypi
 from python_inspector.cli_utils import FileOptionType
 from python_inspector.utils import write_output_in_file
@@ -21,8 +22,7 @@
 
 __version__ = "0.13.0"
 
-DEFAULT_PYTHON_VERSION = "38"
-PYPI_SIMPLE_URL = "https://pypi.org/simple"
+DEFAULT_PYTHON_VERSION = settings.DEFAULT_PYTHON_VERSION
 
 
 def print_version(ctx, param, value):
@@ -71,6 +71,7 @@ def print_version(ctx, param, value):
     "python_version",
     type=click.Choice(utils_pypi.valid_python_versions),
     metavar="PYVER",
+    default=settings.DEFAULT_PYTHON_VERSION,
     show_default=True,
     required=True,
     help="Python version to use for dependency resolution. One of "
@@ -82,6 +83,7 @@ def print_version(ctx, param, value):
     "operating_system",
     type=click.Choice(utils_pypi.PLATFORMS_BY_OS),
     metavar="OS",
+    default=settings.DEFAULT_OS,
     show_default=True,
     required=True,
     help="OS to use for dependency resolution. One of " + ", ".join(utils_pypi.PLATFORMS_BY_OS),
@@ -92,7 +94,7 @@ def print_version(ctx, param, value):
     type=str,
     metavar="INDEX",
     show_default=True,
-    default=tuple([PYPI_SIMPLE_URL]),
+    default=tuple(settings.INDEX_URL),
     multiple=True,
     help="PyPI simple index URL(s) to use in order of preference. "
     "This option can be used multiple times.",
@@ -319,9 +321,6 @@ def get_pretty_options(ctx, generic_paths=False):
         if getattr(param, "hidden", False):
             continue
 
-        if value == param.default:
-            continue
-
         if value in (None, False):
             continue
 

src/python_inspector/settings.py

@@ -0,0 +1,61 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# ScanCode is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/aboutcode-org/python-inspector for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+from pathlib import Path
+
+from pydantic import field_validator
+from pydantic_settings import BaseSettings
+from pydantic_settings import SettingsConfigDict
+
+
+class Settings(BaseSettings):
+    """
+    Reference: https://docs.pydantic.dev/latest/concepts/pydantic_settings/
+    A settings object: use it with an .env file and/or environment variables all prefixed with
+    PYTHON_INSPECTOR_
+    """
+
+    model_config = SettingsConfigDict(
+        env_file=".env",
+        env_file_encoding="utf-8",
+        env_prefix="PYTHON_INSPECTOR_",
+        case_sensitive=True,
+        extra="allow",
+    )
+
+    # the default Python version to use if none is provided
+    DEFAULT_PYTHON_VERSION: str = "39"
+
+    # the default OS to use if none is provided
+    DEFAULT_OS: str = "linux"
+
+    # a list of PyPI simple index URLs. Use a JSON array to represent multiple URLs
+    INDEX_URL: tuple[str, ...] = ("https://pypi.org/simple",)
+
+    # If True, only uses configured INDEX_URLs listed above and ignore other URLs found in requirements
+    USE_ONLY_CONFIGURED_INDEX_URLS: bool = False
+
+    # a path string where to store the cached downloads. Will be created if it does not exists.
+    CACHE_THIRDPARTY_DIR: str = str(Path(Path.home() / ".cache/python_inspector"))
+
+    @field_validator("INDEX_URL")
+    @classmethod
+    def validate_index_url(cls, value):
+        if isinstance(value, str):
+            return (value,)
+        elif isinstance(value, (tuple, list)):
+            return tuple(value)
+        else:
+            raise ValueError(f"INDEX_URL must be either a URL or list of URLs: {value!r}")
+
+
+def create_cache_directory(cache_dir):
+    cache_dir = Path(cache_dir).expanduser().resolve().absolute()
+    if not cache_dir.exists():
+        cache_dir.mkdir(parents=True, exist_ok=True)