Update license clues heuristics

Signed-off-by: Ayan Sinha Mahapatra <[email protected]>

src/licensedcode/detection.py

@@ -15,6 +15,7 @@
 
 import attr
 from license_expression import combine_expressions
+from license_expression import Licensing
 
 from commoncode.resource import clean_path
 from licensedcode.cache import get_index
@@ -96,15 +97,16 @@ class DetectionCategory(Enum):
     UNKNOWN_MATCH = 'unknown-match'
     LICENSE_CLUES = 'license-clues'
     IMPERFECT_COVERAGE = 'imperfect-match-coverage'
-    FALSE_POSITVE = 'false-positive'
+    FALSE_POSITVE = 'possible-false-positive'
     UNDETECTED_LICENSE = 'undetected-license'
 
 
 class DetectionRule(Enum):
     NOT_COMBINED = 'not-combined'
     UNKNOWN_MATCH = 'unknown-match'
     LICENSE_CLUES = 'license-clues'
-    FALSE_POSITIVE = 'false-positive'
+    FALSE_POSITIVE = 'possible-false-positive'
+    NOT_LICENSE_CLUES = 'not-license-clues-as-more-detections-present'
     UNKNOWN_REFERENCE_TO_LOCAL_FILE = 'unknown-reference-to-local-file' 
     UNKNOWN_INTRO_FOLLOWED_BY_MATCH = 'unknown-intro-followed-by-match'
     UNKNOWN_REFERENCE_IN_FILE_TO_PACKAGE = 'unknown-reference-in-file-to-package'
@@ -204,7 +206,10 @@ def from_matches(
         )
 
         if license_expression == None:
-            return cls(matches=matches)
+            return cls(
+                matches=matches,
+                detection_log=detection_log,
+            )
 
         return cls(
             matches=matches,
@@ -440,10 +445,10 @@ def get_detections_from_mappings(detection_mappings):
 
 
 def is_undetected_license_matches(license_matches):
-    
+
     if len(license_matches) != 1:
         return False
-    
+
     if license_matches[0].matcher == MATCHER_UNDETECTED:
         return True
 
@@ -453,11 +458,15 @@ def is_correct_detection(license_matches):
     Return True if all the matches in `license_matches` List of LicenseMatch
     are correct license detections.
     """
-    #TODO: Add matches with full match coverage
     matchers = (license_match.matcher for license_match in license_matches)
+    is_match_coverage_perfect = [
+        license_match.coverage() == 100
+        for license_match in license_matches
+    ]
+
     return (
-        all(matcher in ("1-hash", "1-spdx-id") for matcher in matchers)
-        and not has_unknown_matches(license_matches)
+        all(matcher in ("1-hash", "1-spdx-id", "2-aho") for matcher in matchers)
+        and all(is_match_coverage_perfect) and not has_unknown_matches(license_matches)
     )
 
 
@@ -952,15 +961,15 @@ def analyze_detection(license_matches, package_license=False):
     elif has_unknown_references_to_local_files(license_matches):
         return DetectionCategory.UNKNOWN_FILE_REFERENCE_LOCAL.value
 
-    # Case where all matches have `matcher` as `1-hash` or `4-spdx-id`
-    elif is_correct_detection(license_matches):
-        return DetectionCategory.PERFECT_DETECTION.value
-
     # Case where the match is a false positive
     # In package license detection this is turned off
     elif not package_license and is_false_positive(license_matches, package_license):
         return DetectionCategory.FALSE_POSITVE.value
 
+    # Case where all matches have `matcher` as `1-hash` or `4-spdx-id`
+    elif is_correct_detection(license_matches):
+        return DetectionCategory.PERFECT_DETECTION.value
+
     elif is_license_clues(license_matches):
         return DetectionCategory.LICENSE_CLUES.value
 
@@ -1156,6 +1165,43 @@ def find_referenced_resource(referenced_filename, resource, codebase, **kwargs):
         return resource
 
 
+def process_detections(detections, licensing=Licensing()):
+    """
+    Yield LicenseDetection objects given a list of LicenseDetection objects
+    after postprocessing to include license clues as detections if there are
+    other proper detections with the same license keys. 
+    """
+    if len(detections) == 1:
+        yield detections[0]
+    else:
+        detected_license_keys = set()
+
+        for detection in detections:
+            if detection.license_expression != None:
+                detected_license_keys.update(
+                    licensing.license_keys(detection.license_expression)
+                )
+
+        for detection in detections:
+            if detection.license_expression == None:
+                license_keys = licensing.license_keys(detection.license_expression)
+                if all(
+                    key in detected_license_keys
+                    for key in license_keys
+                ):
+                    detection.license_expression = str(combine_expressions(
+                        expressions=[
+                            match.rule.license_expression
+                            for match in detection.matches
+                        ],
+                        unique=True,
+                        licensing=licensing,
+                    ))
+                    detection.detection_log.append(DetectionRule.NOT_LICENSE_CLUES.value)
+
+            yield detection
+
+
 def detect_licenses(
     index=None,
     location=None,
@@ -1204,10 +1250,15 @@ def detect_licenses(
     if TRACE:
         logger_debug(f"detection: detect_licenses: location: {location}: query_string: {query_string}")
 
+    detections = []
     for group_of_matches in group_matches(matches):
-        yield LicenseDetection.from_matches(
-            matches=group_of_matches,
-            analysis=analysis,
-            post_scan=post_scan,
-            package_license=package_license,
+        detections.append(
+            LicenseDetection.from_matches(
+                matches=group_of_matches,
+                analysis=analysis,
+                post_scan=post_scan,
+                package_license=package_license,
+            )
         )
+
+    yield from process_detections(detections)

src/packagedcode/debian_copyright.py

@@ -950,12 +950,13 @@ def get_license_detections_mapping(self):
             if not license_matches:
                 continue
 
-            detection = LicenseDetection.from_matches(
-                license_matches
+            detection_objects.append(
+                LicenseDetection.from_matches(
+                    matches=license_matches,
+                    package_license=True,
+                )
             )
 
-            detection_objects.append(detection)
-
         detections_mapping, _expression = get_mapping_and_expression_from_detections(
             license_detections=detection_objects,
             whole_lines=False,

tests/licensedcode/data/plugin_license/scan/ffmpeg-license.expected.json

@@ -3,8 +3,8 @@
     {
       "path": "ffmpeg-LICENSE.md",
       "type": "file",
-      "detected_license_expression": "(lgpl-2.1-plus AND other-permissive AND gpl-2.0-plus) AND (lgpl-3.0 AND lgpl-3.0-plus AND (lgpl-3.0 AND gpl-3.0)) AND (ijg AND mit) AND gpl-1.0-plus AND (gpl-2.0 AND apache-2.0 AND lgpl-3.0-plus) AND (gpl-2.0 AND lgpl-2.0-plus AND proprietary-license)",
-      "detected_license_expression_spdx": "(LGPL-2.1-or-later AND LicenseRef-scancode-other-permissive AND GPL-2.0-or-later) AND (LGPL-3.0-only AND LGPL-3.0-or-later AND (LGPL-3.0-only AND GPL-3.0-only)) AND (IJG AND MIT) AND GPL-1.0-or-later AND (GPL-2.0-only AND Apache-2.0 AND LGPL-3.0-or-later) AND (GPL-2.0-only AND LGPL-2.0-or-later AND LicenseRef-scancode-proprietary-license)",
+      "detected_license_expression": "(lgpl-2.1-plus AND other-permissive AND gpl-2.0-plus) AND gpl-1.0-plus AND (lgpl-3.0 AND lgpl-3.0-plus AND (lgpl-3.0 AND gpl-3.0)) AND (ijg AND mit) AND (gpl-2.0 AND apache-2.0 AND lgpl-3.0-plus) AND (gpl-2.0 AND lgpl-2.0-plus AND proprietary-license)",
+      "detected_license_expression_spdx": "(LGPL-2.1-or-later AND LicenseRef-scancode-other-permissive AND GPL-2.0-or-later) AND GPL-1.0-or-later AND (LGPL-3.0-only AND LGPL-3.0-or-later AND (LGPL-3.0-only AND GPL-3.0-only)) AND (IJG AND MIT) AND (GPL-2.0-only AND Apache-2.0 AND LGPL-3.0-or-later) AND (GPL-2.0-only AND LGPL-2.0-or-later AND LicenseRef-scancode-proprietary-license)",
       "license_detections": [
         {
           "license_expression": "lgpl-2.1-plus AND other-permissive AND gpl-2.0-plus",
@@ -86,6 +86,52 @@
             }
           ]
         },
+        {
+          "license_expression": "gpl-1.0-plus",
+          "detection_log": [
+            "possible-false-positive",
+            "not-license-clues-as-more-detections-present"
+          ],
+          "matches": [
+            {
+              "score": 50.0,
+              "start_line": 18,
+              "end_line": 18,
+              "matched_length": 1,
+              "match_coverage": 100.0,
+              "matcher": "2-aho",
+              "license_expression": "gpl-1.0-plus",
+              "rule_identifier": "gpl_bare_word_only.RULE",
+              "referenced_filenames": [],
+              "is_license_text": false,
+              "is_license_notice": false,
+              "is_license_reference": true,
+              "is_license_tag": false,
+              "is_license_intro": false,
+              "rule_length": 1,
+              "rule_relevance": 50,
+              "matched_text": "  libavcodec/x86/flac_dsp_gpl.asm",
+              "licenses": [
+                {
+                  "key": "gpl-1.0-plus",
+                  "name": "GNU General Public License 1.0 or later",
+                  "short_name": "GPL 1.0 or later",
+                  "category": "Copyleft",
+                  "is_exception": false,
+                  "is_unknown": false,
+                  "owner": "Free Software Foundation (FSF)",
+                  "homepage_url": "http://www.gnu.org/licenses/old-licenses/gpl-1.0-standalone.html",
+                  "text_url": "http://www.gnu.org/licenses/old-licenses/gpl-1.0-standalone.html",
+                  "reference_url": "https://scancode-licensedb.aboutcode.org/gpl-1.0-plus",
+                  "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/gpl-1.0-plus.LICENSE",
+                  "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/gpl-1.0-plus.yml",
+                  "spdx_license_key": "GPL-1.0-or-later",
+                  "spdx_url": "https://spdx.org/licenses/GPL-1.0-or-later"
+                }
+              ]
+            }
+          ]
+        },
         {
           "license_expression": "lgpl-3.0 AND lgpl-3.0-plus AND (lgpl-3.0 AND gpl-3.0)",
           "detection_log": [
@@ -664,45 +710,7 @@
           ]
         }
       ],
-      "license_clues": [
-        {
-          "score": 50.0,
-          "start_line": 18,
-          "end_line": 18,
-          "matched_length": 1,
-          "match_coverage": 100.0,
-          "matcher": "2-aho",
-          "license_expression": "gpl-1.0-plus",
-          "rule_identifier": "gpl_bare_word_only.RULE",
-          "referenced_filenames": [],
-          "is_license_text": false,
-          "is_license_notice": false,
-          "is_license_reference": true,
-          "is_license_tag": false,
-          "is_license_intro": false,
-          "rule_length": 1,
-          "rule_relevance": 50,
-          "matched_text": "  libavcodec/x86/flac_dsp_gpl.asm",
-          "licenses": [
-            {
-              "key": "gpl-1.0-plus",
-              "name": "GNU General Public License 1.0 or later",
-              "short_name": "GPL 1.0 or later",
-              "category": "Copyleft",
-              "is_exception": false,
-              "is_unknown": false,
-              "owner": "Free Software Foundation (FSF)",
-              "homepage_url": "http://www.gnu.org/licenses/old-licenses/gpl-1.0-standalone.html",
-              "text_url": "http://www.gnu.org/licenses/old-licenses/gpl-1.0-standalone.html",
-              "reference_url": "https://scancode-licensedb.aboutcode.org/gpl-1.0-plus",
-              "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/gpl-1.0-plus.LICENSE",
-              "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/gpl-1.0-plus.yml",
-              "spdx_license_key": "GPL-1.0-or-later",
-              "spdx_url": "https://spdx.org/licenses/GPL-1.0-or-later"
-            }
-          ]
-        }
-      ],
+      "license_clues": [],
       "percentage_of_license_text": 34.96,
       "scan_errors": []
     }