Refactor license detection to use rehydrated objects

* Due to license references being default, reference data isn't
  inlined anymore, so we need to use the cache to get this data
  and also rehydrate them into objects to be able to post process
  license related info.
* Use license objects wherever possible instead of mappings.

Signed-off-by: Ayan Sinha Mahapatra <ayansmahapatra@gmail.com>

Author: AyanSinhaMahapatra

src/cluecode/plugin_filter_clues.py

@@ -63,10 +63,8 @@ def process_codebase(self, codebase, **kwargs):
 
         from licensedcode.cache import get_index
 
-        rules_by_id = {r.identifier: r for r in get_index().rules_by_rid}
-
         for resource in codebase.walk():
-            filtered = filter_ignorable_resource_clues(resource, rules_by_id)
+            filtered = filter_ignorable_resource_clues(resource, get_index().rules_by_id)
             if filtered:
                 filtered.save(codebase)
 

src/licensedcode/detection.py

@@ -131,6 +131,7 @@ class LicenseIndex(object):
         'digit_only_tids',
         'tokens_by_tid',
 
+        'rules_by_id',
         'rules_by_rid',
         'tids_by_rid',
 
@@ -190,6 +191,9 @@ def __init__(
         # Note: all the following are mappings-like (using lists) of
         # rid-> data are lists of data where the index is the rule id.
 
+        # mapping of rule identifiers -> rule objects 
+        self.rules_by_id = {}
+
         # maping-like of rule_id -> rule objects proper
         self.rules_by_rid = []
 
@@ -305,6 +309,7 @@ def _add_rules(
                 dictionary[sts] = stid
 
         self.rules_by_rid = rules_by_rid = list(rules)
+        self.rules_by_id = {r.identifier: r for r in self.rules_by_rid}
         if TRACE_INDEXING:
             for _rid, _rule in enumerate(rules_by_rid):
                 logger_debug('rules_by_rid:', _rid, _rule)

src/licensedcode/index.py

@@ -22,6 +22,13 @@
 from licensedcode.tokenize import index_tokenizer
 from licensedcode.tokenize import matched_query_text_tokenizer
 
+
+from scancode.api import SPDX_LICENSE_URL
+from scancode.api import SCANCODE_LICENSEDB_URL
+from scancode.api import SCANCODE_LICENSE_URL
+from scancode.api import SCANCODE_LICENSE_RULE_URL
+from scancode.api import SCANCODE_RULE_URL
+
 """
 LicenseMatch data structure and processing.
 A key feature is merging and filtering of matches.
@@ -755,8 +762,8 @@ def matched_text(
 
     def get_mapping(
         self,
-        license_url_template,
-        spdx_license_url,
+        license_url_template=SCANCODE_LICENSEDB_URL,
+        spdx_license_url=SPDX_LICENSE_URL,
         include_text=False,
         license_text_diagnostics=False,
         whole_lines=True,
@@ -777,11 +784,6 @@ def get_mapping(
                 else:
                     matched_text = self.matched_text(whole_lines=False, highlight=False)
 
-        SCANCODE_DATA_BASE_URL = 'https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data'
-        SCANCODE_LICENSE_URL = SCANCODE_DATA_BASE_URL + '/licenses/{}.LICENSE'
-        SCANCODE_LICENSE_RULE_URL = SCANCODE_DATA_BASE_URL + '/licenses/{}'
-        SCANCODE_RULE_URL = SCANCODE_DATA_BASE_URL + '/rules/{}'
-
         result = {}
 
         # Detection Level Information

src/licensedcode/match.py

@@ -45,6 +45,8 @@
 from licensedcode.tokenize import key_phrase_tokenizer
 from licensedcode.tokenize import KEY_PHRASE_OPEN
 from licensedcode.tokenize import KEY_PHRASE_CLOSE
+from scancode.api import SCANCODE_LICENSE_RULE_URL
+from scancode.api import SCANCODE_RULE_URL
 
 """
 Reference License and license Rule structures persisted as a combo of a YAML
@@ -1770,6 +1772,31 @@ def get_min_high_matched_length(self, unique=False):
         return (self.min_high_matched_length_unique if unique
                 else self.min_high_matched_length)
 
+    def get_reference_data(self, matcher=None):
+
+        data = {}
+
+        data['license_expression'] = self.license_expression
+        data['rule_identifier'] = self.identifier
+        if matcher:
+            if matcher == "1-spdx-id":
+                data['rule_url'] = None
+            elif self.is_from_license:
+                data['rule_url'] = SCANCODE_LICENSE_RULE_URL.format(self.identifier)
+            else:
+                data['rule_url'] = SCANCODE_RULE_URL.format(self.identifier)
+
+        data['referenced_filenames'] = self.referenced_filenames
+        data['is_license_text'] = self.is_license_text
+        data['is_license_notice'] = self.is_license_notice
+        data['is_license_reference'] = self.is_license_reference
+        data['is_license_tag'] = self.is_license_tag
+        data['is_license_intro'] = self.is_license_intro
+        data['rule_length'] = self.length
+        data['rule_relevance'] = self.relevance
+
+        return data
+
     def to_dict(self, include_text=False):
         """
         Return an ordered mapping of self, excluding texts unless

src/licensedcode/models.py

@@ -24,14 +24,15 @@
 from licensedcode.detection import get_detected_license_expression
 from licensedcode.detection import get_matches_from_detection_mappings
 from licensedcode.detection import get_referenced_filenames
-from licensedcode.detection import SCANCODE_LICENSEDB_URL
 from licensedcode.detection import LicenseDetection
 from licensedcode.detection import group_matches
 from licensedcode.detection import process_detections
 from licensedcode.detection import DetectionCategory
 from licensedcode.detection import detections_from_license_detection_mappings
 from licensedcode.detection import matches_from_license_match_mappings
 from licensedcode.detection import UniqueDetection
+from licensedcode.detection import LicenseDetectionFromResult
+from licensedcode.licenses_reference import populate_license_references
 from packagedcode.utils import combine_expressions
 from scancode.api import SCANCODE_LICENSEDB_URL
 
@@ -64,11 +65,13 @@ class LicenseScanner(ScanPlugin):
         ('license_detections', attr.ib(default=attr.Factory(list))),
         ('license_clues', attr.ib(default=attr.Factory(list))),
         ('percentage_of_license_text', attr.ib(default=0)),
-        ('for_licenses', attr.ib(default=attr.Factory(list))),
+        ('for_license_detections', attr.ib(default=attr.Factory(list))),
     ])
 
     codebase_attributes = dict(
-        licenses=attr.ib(default=attr.Factory(list)),
+        license_detections=attr.ib(default=attr.Factory(list)),
+        license_references=attr.ib(default=attr.Factory(list)),
+        license_rule_references=attr.ib(default=attr.Factory(list))
     )
 
     sort_order = 2
@@ -209,28 +212,30 @@ def process_codebase(self, codebase, **kwargs):
                     f'after : {license_expressions_after}'
                 )
 
-        populate_for_licenses_in_resources(
+        populate_for_license_detections_in_resources(
             codebase=codebase,
             detections=unique_license_detections,
         )
-        codebase.attributes.licenses.extend([
+        codebase.attributes.license_detections.extend([
             unique_detection.to_dict()
             for unique_detection in unique_license_detections
         ])
 
+        populate_license_references(codebase)
 
-def populate_for_licenses_in_resources(codebase, detections):
+
+def populate_for_license_detections_in_resources(codebase, detections):
 
     for detection in detections:
         if TRACE:
             logger_debug(
-                f'populate_for_licenses_in_resources:',
+                f'populate_for_license_detections_in_resources:',
                 f'for detection: {detection.license_expression}\n',
                 f'file paths: {detection.files}',
             )
         for file_region in detection.files:
             resource = codebase.get_resource(path=file_region.path)
-            resource.for_licenses.append(detection.identifier)
+            resource.for_license_detections.append(detection.identifier)
 
 
 def collect_license_detections(codebase):
@@ -243,7 +248,7 @@ def collect_license_detections(codebase):
 
     if hasattr(codebase.root, 'license_detections'):
         has_licenses = True
-        
+
     all_license_detections = []
 
     for resource in codebase.walk():
@@ -346,16 +351,22 @@ def add_referenced_license_matches_for_detections(resource, codebase):
     if not resource.is_file:
         return
 
-    license_detections = resource.license_detections
-    if not license_detections:
+    license_detection_mappings = resource.license_detections
+    if not license_detection_mappings:
         return
 
     modified = False
 
-    for detection in license_detections:
+    for license_detection_mapping in license_detection_mappings:
+
+        license_detection_object = LicenseDetectionFromResult.from_license_detection_mapping(
+            license_detection_mapping=license_detection_mapping,
+            file_path=resource.path,
+        )
         detection_modified = False
-        matches = detection["matches"]
-        referenced_filenames = get_referenced_filenames(matches)
+        license_match_mappings = license_detection_mapping["matches"]
+        referenced_filenames = get_referenced_filenames(license_detection_object.matches)
+
         if not referenced_filenames:
             continue 
 
@@ -369,7 +380,7 @@ def add_referenced_license_matches_for_detections(resource, codebase):
             if referenced_resource and referenced_resource.license_detections:
                 modified = True
                 detection_modified = True
-                matches.extend(
+                license_match_mappings.extend(
                     get_matches_from_detection_mappings(
                         license_detections=referenced_resource.license_detections
                     )
@@ -379,12 +390,12 @@ def add_referenced_license_matches_for_detections(resource, codebase):
             continue
 
         detection_log, license_expression = get_detected_license_expression(
-            matches=matches,
+            license_match_mappings=license_match_mappings,
             analysis=DetectionCategory.UNKNOWN_FILE_REFERENCE_LOCAL.value,
             post_scan=True,
         )
-        detection["license_expression"] = str(license_expression)
-        detection["detection_log"] = detection_log
+        license_detection_mapping["license_expression"] = str(license_expression)
+        license_detection_mapping["detection_log"] = detection_log
 
     if modified:
         license_expressions = [