Refactor and fix package assembly for installed wheels

src/packagedcode/pypi.py

@@ -13,6 +13,7 @@
 import json
 import logging
 import os
+import posixpath
 import re
 import sys
 import tempfile
@@ -390,19 +391,23 @@ def assign_package_to_resources(cls, package, resource, codebase, package_adder)
         Assign files to package for an installed wheel. This requires a bit
         of navigation around as the files can be in multiple places.
         """
-        site_packages = resource.parent(codebase).parent(codebase)
+        dist_info_dir = resource.parent(codebase)
+        if not dist_info_dir:
+            return
+
+        # This could be a regular directory too which is not `site-packages`
+        site_packages = dist_info_dir.parent(codebase)
         if not site_packages:
             return
+
         package_data = resource.package_data
         assert len(resource.package_data) == 1, (
             f'Unsupported Pypi METADATA wheel structure: {resource.path!r} '
             f'with multiple {package_data!r}'
         )
 
         package_data = models.PackageData.from_dict(package_data[0])
-
         package_uid = package.package_uid
-
         if package_uid:
             # save thyself!
             package_adder(package_uid, resource, codebase)
@@ -414,76 +419,33 @@ def assign_package_to_resources(cls, package, resource, codebase, package_adder)
                 # relative paths need special treatment
                 # most of thense are references to bin ../../../bin/wheel
                 cannot_resolve = False
-                ref_resource = None
+                ref_resource = site_packages
+                # note that resolving leading ".." always stays in the codebase 
                 while path_ref.startswith('..'):
                     _, _, path_ref = path_ref.partition('../')
-                    ref_resource = site_packages.parent(codebase)
+                    ref_resource = ref_resource.parent(codebase)
                     if not ref_resource:
                         cannot_resolve = True
                         break
+
                 if cannot_resolve or not ref_resource:
                     # TODO:w e should log these kind of things
                     continue
                 else:
-                    if package_uid:
+                    ref_resource = codebase.get_resource(
+                        path=posixpath.join(ref_resource.path, path_ref)
+                    )
+                    if ref_resource and package_uid:
                         package_adder(package_uid, ref_resource, codebase)
             else:
-                ref_resource = get_resource_for_path(
-                    path=path_ref,
-                    root=site_packages,
-                    codebase=codebase,
+                # These are absolute paths from the site-packages directory
+                ref_resource = codebase.get_resource(
+                    path=posixpath.join(site_packages.path, path_ref)
                 )
                 if ref_resource and package_uid:
                     package_adder(package_uid, ref_resource, codebase)
 
 
-def get_resource_for_path(path, root, codebase):
-    """
-    Return a resource in ``codebase`` that has a ``path`` relative to the
-    ``root` Resource
-
-    For example, say we start from this:
-        path: this/is/that therefore segments [this, is, that]
-        root: /usr/foo
-
-    We would have these iterations:
-    iteration1
-        root = /usr/foo
-        segments = [this, is, that]
-        seg  this
-        segments = [is, that]
-        children = [/usr/foo/this]
-        root = /usr/foo/this
-
-    iteration2
-        root = /usr/foo/this
-        segments = [is, that]
-        seg  is
-        segments = [that]
-        children = [/usr/foo/this/is]
-        root = /usr/foo/this/is
-
-    iteration3
-        root = /usr/foo/this/is
-        segments = [that]
-        seg  that
-        segments = []
-        children = [/usr/foo/this/is/that]
-        root = /usr/foo/this/is/that
-
-    finally return root as /usr/foo/this/is/that
-    """
-    segments = path.strip('/').split('/')
-    while segments:
-        seg = segments.pop(0)
-        children = [c for c in root.children(codebase) if c.name == seg]
-        if len(children) != 1:
-            return
-        else:
-            root = children[0]
-    return root
-
-
 class PyprojectTomlHandler(BaseExtractedPythonLayout):
     datasource_id = 'pypi_pyproject_toml'
     path_patterns = ('*pyproject.toml',)
@@ -1680,10 +1642,10 @@ def get_declared_license(metainfo):
     # TODO: We should make the declared license as it is, this should be
     # updated in scancode to parse a pure string
     lic = get_attribute(metainfo, 'License')
-
-    license_file = None
-    if lic and 'file' in lic:
-        license_file = lic.pop('file')
+    license_file = get_attribute(metainfo, 'License-File')
+    if not license_file and lic:
+        if isinstance(lic, dict) and 'file' in lic.keys():
+            license_file = lic.pop('file')
 
     if lic and not lic == 'UNKNOWN':
         if 'text' in lic:

tests/packagedcode/data/conda/assembly-conda-scan.json

@@ -177,7 +177,8 @@
       "is_private": false,
       "is_virtual": false,
       "extra_data": {
-        "Documentation": "https://requests.readthedocs.io"
+        "Documentation": "https://requests.readthedocs.io",
+        "license_file": "LICENSE"
       },
       "repository_homepage_url": "https://pypi.org/project/requests",
       "repository_download_url": "https://pypi.org/packages/source/r/requests/requests-2.32.3.tar.gz",
@@ -491,7 +492,8 @@
           "is_private": false,
           "is_virtual": false,
           "extra_data": {
-            "Documentation": "https://requests.readthedocs.io"
+            "Documentation": "https://requests.readthedocs.io",
+            "license_file": "LICENSE"
           },
           "dependencies": [
             {

tests/packagedcode/data/instance/python-package-instance-expected-with-test-manifests-with-license.json

@@ -77,7 +77,8 @@
       "is_private": false,
       "is_virtual": false,
       "extra_data": {
-        "Documentation": "https://setuptools.readthedocs.io/"
+        "Documentation": "https://setuptools.readthedocs.io/",
+        "license_file": "LICENSE"
       },
       "repository_homepage_url": "https://pypi.org/project/setuptools",
       "repository_download_url": "https://pypi.org/packages/source/s/setuptools/setuptools-58.2.0.tar.gz",
@@ -308,7 +309,8 @@
           "is_private": false,
           "is_virtual": false,
           "extra_data": {
-            "Documentation": "https://setuptools.readthedocs.io/"
+            "Documentation": "https://setuptools.readthedocs.io/",
+            "license_file": "LICENSE"
           },
           "dependencies": [],
           "repository_homepage_url": "https://pypi.org/project/setuptools",

tests/packagedcode/data/instance/python-package-instance-expected-with-test-manifests.json

@@ -77,7 +77,8 @@
       "is_private": false,
       "is_virtual": false,
       "extra_data": {
-        "Documentation": "https://setuptools.readthedocs.io/"
+        "Documentation": "https://setuptools.readthedocs.io/",
+        "license_file": "LICENSE"
       },
       "repository_homepage_url": "https://pypi.org/project/setuptools",
       "repository_download_url": "https://pypi.org/packages/source/s/setuptools/setuptools-58.2.0.tar.gz",
@@ -194,7 +195,8 @@
           "is_private": false,
           "is_virtual": false,
           "extra_data": {
-            "Documentation": "https://setuptools.readthedocs.io/"
+            "Documentation": "https://setuptools.readthedocs.io/",
+            "license_file": "LICENSE"
           },
           "dependencies": [],
           "repository_homepage_url": "https://pypi.org/project/setuptools",

tests/packagedcode/data/pypi/site-packages/codebase/lib/python3.9/site-packages/click-8.0.4.dist-info/RECORD

@@ -5,37 +5,4 @@ click-8.0.4.dist-info/RECORD,,
 click-8.0.4.dist-info/WHEEL,sha256=G16H4A3IeoQmnOrYV4ueZGKSjhipXx8zc8nu9FGlvMA,92
 click-8.0.4.dist-info/top_level.txt,sha256=J1ZQogalYS4pphY_lPECoNMfw0HzTSrZglC4Yfwo4xA,6
 click/__init__.py,sha256=bOKrvMqmR9rN07vN_ycyrdF-EcTCl-EmuAjq-Fp4yPM,3243
-click/__pycache__/__init__.cpython-36.pyc,,
-click/__pycache__/_compat.cpython-36.pyc,,
-click/__pycache__/_termui_impl.cpython-36.pyc,,
-click/__pycache__/_textwrap.cpython-36.pyc,,
-click/__pycache__/_unicodefun.cpython-36.pyc,,
-click/__pycache__/_winconsole.cpython-36.pyc,,
-click/__pycache__/core.cpython-36.pyc,,
-click/__pycache__/decorators.cpython-36.pyc,,
-click/__pycache__/exceptions.cpython-36.pyc,,
-click/__pycache__/formatting.cpython-36.pyc,,
-click/__pycache__/globals.cpython-36.pyc,,
-click/__pycache__/parser.cpython-36.pyc,,
-click/__pycache__/shell_completion.cpython-36.pyc,,
-click/__pycache__/termui.cpython-36.pyc,,
-click/__pycache__/testing.cpython-36.pyc,,
-click/__pycache__/types.cpython-36.pyc,,
-click/__pycache__/utils.cpython-36.pyc,,
-click/_compat.py,sha256=JIHLYs7Jzz4KT9t-ds4o4jBzLjnwCiJQKqur-5iwCKI,18810
-click/_termui_impl.py,sha256=qK6Cfy4mRFxvxE8dya8RBhLpSC8HjF-lvBc6aNrPdwg,23451
-click/_textwrap.py,sha256=10fQ64OcBUMuK7mFvh8363_uoOxPlRItZBmKzRJDgoY,1353
-click/_unicodefun.py,sha256=JKSh1oSwG_zbjAu4TBCa9tQde2P9FiYcf4MBfy5NdT8,3201
-click/_winconsole.py,sha256=5ju3jQkcZD0W27WEMGqmEP4y_crUVzPCqsX_FYb7BO0,7860
-click/core.py,sha256=MYYmvqVa_dh4x4f-mr7VyNi6197ucYbJ0fsWW5EnCrA,111432
-click/decorators.py,sha256=5ZngOD72Flo8VLN29iarI0A5u_F-dxmqtUqx4KN8hOg,14879
-click/exceptions.py,sha256=7gDaLGuFZBeCNwY9ERMsF2-Z3R9Fvq09Zc6IZSKjseo,9167
-click/formatting.py,sha256=Frf0-5W33-loyY_i9qrwXR8-STnW3m5gvyxLVUdyxyk,9706
-click/globals.py,sha256=TP-qM88STzc7f127h35TD_v920FgfOD2EwzqA0oE8XU,1961
-click/parser.py,sha256=cAEt1uQR8gq3-S9ysqbVU-fdAZNvilxw4ReJ_T1OQMk,19044
-click/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-click/shell_completion.py,sha256=HHH0wMwlDW4WQhs8rRcS9M8MAo9gltGBN6V2PEbsTp0,18003
-click/termui.py,sha256=mZ12uc3-loFaV__vr8buxn9uIjAhy7QwVuZOQ8jDdjc,28873
-click/testing.py,sha256=ptpMYgRY7dVfE3UDgkgwayu9ePw98sQI3D7zZXiCpj4,16063
-click/types.py,sha256=rj2g3biAXKkNKV8vlwbIKSUlixhXybH84N84fwCYqUU,35092
-click/utils.py,sha256=M8tuplcFFHROha3vQ60ZRSakSB_ng6w9e8Uc1AugPZ0,18934
+click/core.py,sha256=MYYmvqVa_dh4x4f-mr7VyNi6197ucYbJ0fsWW5EnCrA,111432

tests/packagedcode/data/pypi/site-packages/codebase/lib/python3.9/site-packages/pip-20.2.2.dist-info/METADATA

@@ -0,0 +1,34 @@
+Metadata-Version: 2.1
+Name: pip
+Version: 20.2.2
+Summary: The PyPA recommended tool for installing Python packages.
+Home-page: https://pip.pypa.io/
+Author: The pip developers
+Author-email: [email protected]
+License: MIT
+Project-URL: Documentation, https://pip.pypa.io
+Project-URL: Source, https://github.com/pypa/pip
+Project-URL: Changelog, https://pip.pypa.io/en/stable/news/
+Keywords: distutils easy_install egg setuptools wheel virtualenv
+Platform: UNKNOWN
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Topic :: Software Development :: Build Tools
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 2
+Classifier: Programming Language :: Python :: 2.7
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.5
+Classifier: Programming Language :: Python :: 3.6
+Classifier: Programming Language :: Python :: 3.7
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: Implementation :: CPython
+Classifier: Programming Language :: Python :: Implementation :: PyPy
+Requires-Python: >=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*
+
+pip - The Python Package Installer
+==================================
+
+pip is the `package installer`_ for Python. You can use pip to install packages from the `Python Package Index`_ and other indexes.
+

tests/packagedcode/data/pypi/site-packages/codebase/lib/python3.9/site-packages/pip-20.2.2.dist-info/RECORD

@@ -0,0 +1,17 @@
+pip/__init__.py,sha256=fZ401OZGdv55YGsuCEhsvdNa4YCZLmK457SN3qrstJk,455
+pip/__main__.py,sha256=bqCAM1cj1HwHCDx3WJa-LJxOBXimGxE8OjBqAvnhVg0,911
+pip/_internal/__init__.py,sha256=2si23JBW1erg19xIJ8CD6tfGknz0ijtXmzuXjGfGMGE,495
+pip/_internal/main.py,sha256=IVBnUQ-FG7DK6617uEXRB5_QJqspAsBFmTmTesYkbdQ,437
+pip/_internal/cli/main.py,sha256=Hxc9dZyW3xiDsYZX-_J2cGXT5DWNLNn_Y7o9oUme-Ec,2616
+pip-20.2.2.dist-info/LICENSE.txt,sha256=W6Ifuwlk-TatfRU2LR7W1JMcyMj5_y1NkRkOEJvnRDE,1090
+pip-20.2.2.dist-info/METADATA,sha256=ZMrQjU30hfRVRViE4qoVrwZzDSRyGCT8OdkDkkTMBv8,3708
+pip-20.2.2.dist-info/WHEEL,sha256=kGT74LWyRUZrL4VgLh6_g12IeVl_9u9ZVhadrgXZUEY,110
+pip-20.2.2.dist-info/entry_points.txt,sha256=HtfDOwpUlr9s73jqLQ6wF9V0_0qvUXJwCBz7Vwx0Ue0,125
+pip-20.2.2.dist-info/top_level.txt,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4
+pip-20.2.2.dist-info/RECORD,,
+../../../bin/pip-3.8,,
+../../../bin/pip3,,
+pip-20.2.2.dist-info/__pycache__,,
+../../../bin/pip,,
+pip-20.2.2.virtualenv,,
+../../../bin/pip3.8,,