Add --licenses-reference option

Add new command line option to not inline license, licenseDB and license
detection level information.

Signed-off-by: Ayan Sinha Mahapatra <[email protected]>

Author: AyanSinhaMahapatra

src/licensedcode/detection.py

@@ -7,6 +7,7 @@
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
 
+import sys
 import os
 import logging
 from enum import Enum
@@ -95,6 +96,18 @@ class CombinationReason(Enum):
     UNVERSIONED_FOLLOWED_BY_VERSIONED = 'un-versioned-followed-by-versioned'
 
 
+@attr.s
+class FileRegion:
+    """
+    A file has one or more file-regions, which are separate regions of the file
+    containing some license information (separated by code/text/others in between),
+    and identified by a start line and an end line.
+    """
+    path = attr.ib(type=str)
+    start_line = attr.ib(type=int)
+    end_line = attr.ib(type=int)
+
+
 @attr.s(slots=True, eq=False, order=False)
 class LicenseDetection:
     """
@@ -135,6 +148,14 @@ class LicenseDetection:
         )
     )
 
+    # Only used in unique detection calculation and referencing
+    file_region = attr.ib(
+        default=attr.Factory(dict),
+        metadata=dict(
+            help='File path and start end lines to locate the detection.'
+        )
+    )
+
     @classmethod
     def from_matches(cls, matches):
         """
@@ -162,6 +183,19 @@ def from_matches(cls, matches):
             spdx_license_expression=str(spdx_license_expression),
             combination_reasons=reasons,
         )
+    
+    @classmethod
+    def from_mapping(cls, detection):
+        """
+        Return a LicenseDetection created out of `detection` dict with
+        the same attributes.
+        """
+        return cls(
+            matches=detection['matches'],
+            license_expression=detection['license_expression'],
+            spdx_license_expression=detection['spdx_license_expression'],
+            combination_reasons=detection['combination_reasons'],
+        )
 
     def __eq__(self, other):
         return (
@@ -181,36 +215,65 @@ def query(self):
     def qspans(self):
         return [match.qspan for match in self.matches]
 
+    def get_file_region(self, path):
+        """
+        This is an identifier for a license detection, based on it's underlying
+        license matches.
+        """
+        start_line, end_line = self.get_start_end_line()
+        return FileRegion(
+            path=path,
+            start_line=start_line,
+            end_line=end_line,
+        )
+
     @property
     def identifier(self):
         """
         This is an identifier for a license detection, based on it's underlying
         license matches.
+
+        This is not guaranteed to be unique for a detection, as certain small and
+        unknown matches could have the same value for this identifier, but will
+        be unique for most other cases.
         """
         data = []
-        for license_match in self.original_licenses:
-            identifier = (license_match.rule_identifier, license_match.coverage(),)
+        for match in self.matches:
+            identifier = (match['licensedb_identifier'], match['match_coverage'],)
             data.append(identifier)
 
-        return tuple(data)
+        # Return a positive hash value for the tuple
+        return tuple(data).__hash__() % ((sys.maxsize + 1) * 2)
 
     @property
     def identifier_with_text(self):
         """
-        This is an identifier for a issue, which is an unknown license intro,
-        based on it's underlying license matches.
+        This is an identifier for a license detection, based on it's underlying
+        license matches with the tokenized matched_text.
+
+        This is guaranteed to be unique for a detection.
         """
         data = []
-        for license_match in self.original_licenses:
-            tokenized_matched_text = tuple(query_tokenizer(license_match.matched_text))
+        for match in self.matches:
+            tokenized_matched_text = tuple(query_tokenizer(match['matched_text']))
             identifier = (
-                license_match.rule_identifier,
-                license_match.coverage(),
+                match['licensedb_identifier'],
+                match['match_coverage'],
                 tokenized_matched_text,
             )
             data.append(identifier)
 
-        return tuple(data)
+        # Return a positive hash value for the tuple
+        return tuple(data).__hash__() % ((sys.maxsize + 1) * 2)
+    
+    def get_start_end_line(self):
+        """
+        Returns start and end line for a license detection issue, from the
+        license match(es).
+        """
+        start_line = min([match['start_line'] for match in self.matches])
+        end_line = max([match['end_line'] for match in self.matches])
+        return start_line, end_line
 
     def rules_length(self):
         """
@@ -373,6 +436,7 @@ def to_dict(
 
         detection = attr.asdict(self)
         detection["matches"] = data_matches
+        _file_region = detection.pop('file_region')
 
         return detection
 

src/licensedcode/plugin_licenses_reference.py

@@ -8,12 +8,16 @@
 #
 
 import attr
+from collections import Counter
+
 from commoncode.cliutils import PluggableCommandLineOption
 from commoncode.cliutils import POST_SCAN_GROUP
 from license_expression import Licensing
 from plugincode.post_scan import PostScanPlugin
 from plugincode.post_scan import post_scan_impl
 
+from licensedcode.detection import LicenseDetection
+
 # Set to True to enable debug tracing
 TRACE = False
 
@@ -39,7 +43,11 @@ class LicensesReference(PostScanPlugin):
     """
     Add a reference list of all licenses data and text.
     """
-    codebase_attributes = dict(licenses_reference=attr.ib(default=attr.Factory(list)))
+    codebase_attributes = dict(
+        license_references=attr.ib(default=attr.Factory(list)),
+        licensedb_references=attr.ib(default=attr.Factory(list)),
+        license_detection_references=attr.ib(default=attr.Factory(list))
+    )
 
     sort_order = 500
 
@@ -55,30 +63,187 @@ def is_enabled(self, licenses_reference, **kwargs):
         return licenses_reference
 
     def process_codebase(self, codebase, licenses_reference, **kwargs):
-        from licensedcode.cache import get_licenses_db
-        licensing = Licensing()
-
-        license_keys = set()
+        """
+        Get Licenses and LicenseDB data from all license detections in a codebase level list
+        and only refer to them in the resource level detections. 
+        """
+        licexps = []
+        license_db_data = []
+        license_detections_by_path = {}
 
         for resource in codebase.walk():
-            licexps = getattr(resource, 'license_expressions', []) or []
-            for expression in licexps:
-                if expression:
-                    license_keys.update(licensing.license_keys(expression))
-
-        packages = getattr(codebase, 'packages', []) or []
-        for package in packages:
-            # FXIME: license_expression attribute name is changing soon
-            expression = package.get('license_expression')
-            if expression:
-                license_keys.update(licensing.license_keys(expression))
-
-                resource.save(codebase)
-
-        db = get_licenses_db()
-        for key in sorted(license_keys):
-            license_details = db[key].to_dict(
-                include_ignorables=False,
-                include_text=True,
+
+            # Get license_expressions from both package and license detections
+            license_licexps = getattr(resource, 'license_expressions', []) or []
+            package_data = getattr(resource, 'package_data', []) or []
+            # TODO: license_expression attribute name is changing soon
+            package_licexps = [pkg['license_expression'] for pkg in package_data]
+            licexps.extend(license_licexps + package_licexps)
+
+            # Get license matches from both package and license detections
+            licence_detections = getattr(resource, 'licenses', []) or []
+            #TODO: report license detections (with license matches) for packages
+            license_db_data.extend(
+                get_license_db_reference_data(licence_detections=licence_detections)
+            )
+
+            license_detections_by_path[resource.path] = licence_detections
+            codebase.save_resource(resource)
+
+        license_references = get_license_references(license_expressions=licexps)
+        codebase.attributes.license_references.extend(license_references)
+
+        licensedb_references = get_licensedb_references(license_db_data=license_db_data)
+        codebase.attributes.licensedb_references.extend(licensedb_references)
+
+        license_detection_references = get_license_detection_references(license_detections_by_path)
+        codebase.attributes.license_detection_references.extend(license_detection_references)
+
+
+def get_license_references(license_expressions, licensing=Licensing()):
+    """
+    Get a list of License data from a list of `license_expression` strings.
+    """
+    from licensedcode.cache import get_licenses_db
+
+    license_keys = set()
+    license_references = []
+
+    for expression in license_expressions:
+        if expression:
+            license_keys.update(licensing.license_keys(expression))
+
+    db = get_licenses_db()
+    for key in sorted(license_keys):
+        license_references.append(
+            db[key].to_dict(include_ignorables=False, include_text=True)
+        )
+
+    return license_references
+
+
+def get_licensedb_references(license_db_data):
+    """
+    """
+    licence_db_ids = set()
+    licensedb_references = []
+
+    for licdb_ref in license_db_data:
+
+        licdb_id = licdb_ref['licensedb_identifier']
+        if licdb_id not in licence_db_ids:
+            licence_db_ids.update(licdb_id)
+            licensedb_references.append(licdb_ref)
+
+    return licensedb_references
+
+
+def get_license_db_reference_data(licence_detections):
+    """
+    """
+    licence_db_ids = set()
+    license_db_reference_data = []
+
+    for detection in licence_detections:
+        matches = detection['matches']
+
+        for match in matches:
+
+            licdb_id = match['licensedb_identifier']
+
+            ref_data = {}
+            ref_data['license_expression'] = match['license_expression']
+            ref_data['licensedb_identifier'] = licdb_id
+            ref_data['referenced_filenames'] = match.pop('referenced_filenames')
+            ref_data['is_license_text'] = match.pop('is_license_text')
+            ref_data['is_license_notice'] = match.pop('is_license_notice')
+            ref_data['is_license_reference'] = match.pop('is_license_reference')
+            ref_data['is_license_tag'] = match.pop('is_license_tag')
+            ref_data['is_license_intro'] = match.pop('is_license_intro')
+            ref_data['rule_length'] = match.pop('rule_length')
+            ref_data['rule_relevance'] = match.pop('rule_relevance')
+
+            if 'matched_text' in match:
+                ref_data['matched_text'] = match.pop('matched_text')
+
+            _ = match.pop('licenses')
+
+            if licdb_id not in licence_db_ids:
+                licence_db_ids.update(licdb_id)
+                license_db_reference_data.append(ref_data)
+
+    return license_db_reference_data
+
+
+def get_license_detection_references(license_detections_by_path):
+    """
+    """
+    detection_objects = []
+
+    for path, detections in license_detections_by_path.items():
+
+        for detection in detections:
+            detection_obj = LicenseDetection.from_mapping(detection=detection)
+            _matches = detection.pop('matches')
+            _reasons = detection.pop('combination_reasons')
+            detection_obj.file_region = detection_obj.get_file_region(path=path)
+            detection["id"] = detection_obj.identifier
+
+            detection_objects.append(detection_obj)
+
+    detection_references = UniqueDetection.get_unique_detections(detection_objects)
+    return detection_references
+
+
+@attr.s
+class UniqueDetection:
+    """
+    An unique License Detection.
+    """
+    unique_identifier = attr.ib(type=int)
+    license_detection = attr.ib()
+    files = attr.ib(factory=list)
+
+    @classmethod
+    def get_unique_detections(cls, license_detections):
+        """
+        Get all unique license detections from a list of
+        LicenseDetections.
+        """
+        identifiers = get_identifiers(license_detections)
+        unique_detection_counts = dict(Counter(identifiers))
+
+        unique_license_detections = []
+        for detection_identifier in unique_detection_counts.keys():
+            file_regions = (
+                detection.file_region
+                for detection in license_detections
+                if detection_identifier == detection.identifier
+            )
+            all_detections = (
+                detection
+                for detection in license_detections
+                if detection_identifier == detection.identifier
             )
-            codebase.attributes.licenses_reference.append(license_details)
+
+            detection = next(all_detections)
+            unique_license_detections.append(
+                cls(
+                    files=list(file_regions),
+                    license_detection=attr.asdict(detection),
+                    unique_identifier=detection.identifier,
+                )
+            )
+
+        return unique_license_detections
+
+
+def get_identifiers(license_detections):
+    """
+    Get identifiers for all license detections.
+    """
+    identifiers = (
+        detection.identifier
+        for detection in license_detections
+    )
+    return identifiers