Update license rules reference data

* Add rule text to reference data
* Add rule url to reference data
* make rule references unique
* reorder rule references data
* regenerate test expectations

Signed-off-by: Ayan Sinha Mahapatra <ayansmahapatra@gmail.com>

Author: AyanSinhaMahapatra

src/licensedcode/detection.py

@@ -26,16 +26,17 @@
 from licensedcode.cache import get_cache
 from licensedcode.match import LicenseMatch
 from licensedcode.match import set_matched_lines
-from licensedcode.models import Rule
+from licensedcode.models import UnDetectedRule
 from licensedcode.models import BasicRule
-from licensedcode.models import SpdxRule
 from licensedcode.models import compute_relevance
+from licensedcode.models import get_rule_object_from_match
 from licensedcode.spans import Span
 from licensedcode.tokenize import query_tokenizer
 from licensedcode.query import Query
 from licensedcode.query import LINES_THRESHOLD
 from licensedcode.licenses_reference import extract_license_rules_reference_data
 
+
 """
 LicenseDetection data structure and processing.
 
@@ -612,24 +613,8 @@ def matches_from_license_match_mappings(license_match_mappings):
     license_matches = []
 
     for license_match_mapping in license_match_mappings:
-        matcher = license_match_mapping["matcher"]
-        rule_identifier = license_match_mapping["rule_identifier"]
-        if matcher == "1-spdx-id":
-            rule = SpdxRule(
-                license_expression=license_match_mapping["license_expression"],
-                text=license_match_mapping.get("matched_text", None),
-                length=license_match_mapping["matched_length"],
-            )
-        elif rule_identifier == 'package-manifest-unknown':
-            rule = UnDetectedRule(
-                license_expression=license_match_mapping["license_expression"],
-                text=license_match_mapping.get("matched_text", None),
-                length=license_match_mapping["matched_length"],
-            )
-        else:
-            rule = get_index().rules_by_id[rule_identifier]
-
-        license_rule_reference = rule.get_reference_data(matcher=matcher)
+        rule = get_rule_object_from_match(license_match_mapping)
+        license_rule_reference = rule.get_reference_data()
         license_matches.append(
             LicenseMatchFromResult.from_license_match_mapping(
                 license_match_mapping=license_match_mapping,
@@ -1182,35 +1167,6 @@ def get_undetected_matches(query_string):
     return matches
 
 
-@attr.s(slots=True, repr=False)
-class UnDetectedRule(Rule):
-    """
-    A specialized rule object that is used for the special case of extracted
-    license statements without any valid license detection.
-
-    Since there is a license where there is a non empty extracted license
-    statement (typically found in a package manifest), if there is no license
-    detected by scancode, it would be incorrect to not point out that there
-    is a license (though undetected).
-    """
-
-    def __attrs_post_init__(self, *args, **kwargs):
-        self.identifier = 'package-manifest-' + self.license_expression
-        expression = self.licensing.parse(self.license_expression)
-        self.license_expression = expression.render()
-        self.license_expression_object = expression
-        self.is_license_tag = True
-        self.is_small = False
-        self.relevance = 100
-        self.has_stored_relevance = True
-
-    def load(self):
-        raise NotImplementedError
-
-    def dump(self):
-        raise NotImplementedError
-
-
 def get_matches_from_detections(license_detections):
     """
     Return a `license_matches` list of LicenseMatch objects from a

src/licensedcode/licenses_reference.py

@@ -11,6 +11,8 @@
 import logging
 from license_expression import Licensing
 
+from licensedcode.models import get_rule_object_from_match
+
 
 TRACE_REFERENCE = os.environ.get('SCANCODE_DEBUG_LICENSE_REFERENCE', False)
 TRACE_EXTRACT = os.environ.get('SCANCODE_DEBUG_LICENSE_REFERENCE_EXTRACT', False)
@@ -208,17 +210,13 @@ def get_unique_rule_references(rules_data):
     """
     Get a list of unique Rule data from a list of Rule data.
     """
-    rule_identifiers = set()
-    rules_references = []
+    rules_references_by_identifier = {}
 
     for rule_data in rules_data:
-
         rule_identifier = rule_data['rule_identifier']
-        if rule_identifier not in rule_identifiers:
-            rule_identifiers.update(rule_identifier)
-            rules_references.append(rule_data)
+        rules_references_by_identifier[rule_identifier] = rule_data
 
-    return rules_references
+    return rules_references_by_identifier.values()
 
 
 def extract_license_rules_reference_data(license_detections=None, license_matches=None):
@@ -273,17 +271,21 @@ def extract_license_rules_reference_data(license_detections=None, license_matche
 
 def get_reference_data(match):
 
+    rule = get_rule_object_from_match(license_match_mapping=match)
+
     ref_data = {}
-    ref_data['license_expression'] = match['license_expression']
     ref_data['rule_identifier'] = match['rule_identifier']
-    ref_data['referenced_filenames'] = match.pop('referenced_filenames')
+    ref_data['license_expression'] = match['license_expression']
+    ref_data['rule_url'] = rule.rule_url
+    ref_data['rule_relevance'] = match.pop('rule_relevance')
+    ref_data['rule_length'] = match.pop('rule_length')
     ref_data['is_license_text'] = match.pop('is_license_text')
     ref_data['is_license_notice'] = match.pop('is_license_notice')
     ref_data['is_license_reference'] = match.pop('is_license_reference')
     ref_data['is_license_tag'] = match.pop('is_license_tag')
     ref_data['is_license_intro'] = match.pop('is_license_intro')
-    ref_data['rule_length'] = match.pop('rule_length')
-    ref_data['rule_relevance'] = match.pop('rule_relevance')
+    ref_data['referenced_filenames'] = match.pop('referenced_filenames')
+    ref_data['rule_text'] = rule.text
 
     _ = match.pop('licenses')
 

src/licensedcode/models.py

@@ -33,6 +33,7 @@
 from licensedcode import MIN_MATCH_HIGH_LENGTH
 from licensedcode import MIN_MATCH_LENGTH
 from licensedcode import SMALL_RULE
+from licensedcode.cache import get_index
 from licensedcode.frontmatter import SaneYAMLHandler
 from licensedcode.frontmatter import FrontmatterPost
 from licensedcode.frontmatter import dumps_frontmatter
@@ -1577,6 +1578,18 @@ class BasicRule:
             'position is using the magic -1 key.')
     )
 
+    @property
+    def rule_url(self):
+        if 'spdx-license-identifier' in self.identifier:
+            return None
+        elif self.identifier == 'package-manifest-unknown':
+            return None
+        elif self.is_from_license:
+            return SCANCODE_LICENSE_RULE_URL.format(self.identifier)
+        else:
+            return SCANCODE_RULE_URL.format(self.identifier)
+
+
     def rule_file(
         self,
         rules_data_dir=rules_data_dir,
@@ -1772,20 +1785,13 @@ def get_min_high_matched_length(self, unique=False):
         return (self.min_high_matched_length_unique if unique
                 else self.min_high_matched_length)
 
-    def get_reference_data(self, matcher=None):
+    def get_reference_data(self):
 
         data = {}
 
         data['license_expression'] = self.license_expression
         data['rule_identifier'] = self.identifier
-        if matcher:
-            if matcher == "1-spdx-id":
-                data['rule_url'] = None
-            elif self.is_from_license:
-                data['rule_url'] = SCANCODE_LICENSE_RULE_URL.format(self.identifier)
-            else:
-                data['rule_url'] = SCANCODE_RULE_URL.format(self.identifier)
-
+        data['rule_url'] = self.rule_url
         data['referenced_filenames'] = self.referenced_filenames
         data['is_license_text'] = self.is_license_text
         data['is_license_notice'] = self.is_license_notice
@@ -2228,6 +2234,24 @@ def set_relevance(self):
             self.relevance = computed_relevance
 
 
+def get_rule_object_from_match(license_match_mapping):
+    rule_identifier = license_match_mapping["rule_identifier"]
+    if 'spdx-license-identifier' in rule_identifier:
+        return SpdxRule(
+            license_expression=license_match_mapping["license_expression"],
+            text=license_match_mapping.get("matched_text", None),
+            length=license_match_mapping["matched_length"],
+        )
+    elif rule_identifier == 'package-manifest-unknown':
+        return UnDetectedRule(
+            license_expression=license_match_mapping["license_expression"],
+            text=license_match_mapping.get("matched_text", None),
+            length=license_match_mapping["matched_length"],
+        )
+    else:
+        return get_index().rules_by_id[rule_identifier]
+
+
 def compute_relevance(length):
     """
     Return a computed ``relevance`` given a ``length`` and a threshold.
@@ -2445,6 +2469,35 @@ def compute_unique_id(self):
         return hashlib.md5(self.text.encode('utf-8')).hexdigest()
 
 
+@attr.s(slots=True, repr=False)
+class UnDetectedRule(Rule):
+    """
+    A specialized rule object that is used for the special case of extracted
+    license statements without any valid license detection.
+
+    Since there is a license where there is a non empty extracted license
+    statement (typically found in a package manifest), if there is no license
+    detected by scancode, it would be incorrect to not point out that there
+    is a license (though undetected).
+    """
+
+    def __attrs_post_init__(self, *args, **kwargs):
+        self.identifier = 'package-manifest-' + self.license_expression
+        expression = self.licensing.parse(self.license_expression)
+        self.license_expression = expression.render()
+        self.license_expression_object = expression
+        self.is_license_tag = True
+        self.is_small = False
+        self.relevance = 100
+        self.has_stored_relevance = True
+
+    def load(self):
+        raise NotImplementedError
+
+    def dump(self):
+        raise NotImplementedError
+
+
 def _print_rule_stats():
     """
     Print rules statistics.

tests/cluecode/data/plugin_filter_clues/filtered-expected.json

@@ -47,16 +47,18 @@
   ],
   "license_rule_references": [
     {
-      "license_expression": "apache-1.1",
       "rule_identifier": "apache-1.1_63.RULE",
-      "referenced_filenames": [],
+      "license_expression": "apache-1.1",
+      "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/apache-1.1_63.RULE",
+      "rule_relevance": 100,
+      "rule_length": 367,
       "is_license_text": true,
       "is_license_notice": false,
       "is_license_reference": false,
       "is_license_tag": false,
       "is_license_intro": false,
-      "rule_length": 367,
-      "rule_relevance": 100
+      "referenced_filenames": [],
+      "rule_text": "is licensed under the\nApache Software License, Version 1.1, which is reproduced below.\n\n/*\n* The Apache Software License, Version 1.1\n*\n*\n* Copyright (c)  The Apache Software Foundation. All rights\n* reserved.\n*\n* Redistribution and use in source and binary forms, with or without\n* modification, are permitted provided that the following conditions\n* are met:\n*\n* 1. Redistributions of source code must retain the above copyright\n* notice, this list of conditions and the following disclaimer.\n*\n* 2. Redistributions in binary form must reproduce the above copyright\n* notice, this list of conditions and the following disclaimer in\n* the documentation and/or other materials provided with the\n* distribution.\n*\n* 3. The end-user documentation included with the redistribution,\n* if any, must include the following acknowledgment:\n* \"This product includes software developed by the\n* Apache Software Foundation (http://www.apache.org/).\"\n* Alternately, this acknowledgment may appear in the software itself,\n* if and wherever such third-party acknowledgments normally appear.\n*\n* 4. The names \"Xerces\" and \"Apache Software Foundation\" must\n* not be used to endorse or promote products derived from this\n* software without prior written permission. For written\n* permission, please contact apache@apache.org.\n*\n* 5. Products derived from this software may not be called \"Apache\",\n* nor may \"Apache\" appear in their name, without prior written\n* permission of the Apache Software Foundation.\n*\n* THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED\n* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES\n* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE\n* DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR\n* ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,\n* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT\n* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF\n* USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\n* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,\n* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT\n* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF\n* SUCH DAMAGE.\n* ====================================================================\n*\n* This software consists of voluntary contributions made by many\n* individuals on behalf of the Apache Software Foundation and was\n* originally based on software copyright (c) 1999, International\n* Business Machines, Inc., http://www.ibm.com. For more\n* information on the Apache Software Foundation, please see\n* <http://www.apache.org/>."
     }
   ],
   "files": [

tests/cluecode/data/plugin_filter_clues/filtered-expected2.json

@@ -39,16 +39,18 @@
   ],
   "license_rule_references": [
     {
-      "license_expression": "pygres-2.2",
       "rule_identifier": "pygres-2.2_2.RULE",
-      "referenced_filenames": [],
+      "license_expression": "pygres-2.2",
+      "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/pygres-2.2_2.RULE",
+      "rule_relevance": 100,
+      "rule_length": 145,
       "is_license_text": true,
       "is_license_notice": false,
       "is_license_reference": false,
       "is_license_tag": false,
       "is_license_intro": false,
-      "rule_length": 145,
-      "rule_relevance": 100
+      "referenced_filenames": [],
+      "rule_text": "Permission to use, copy, modify, and distribute this software and its\ndocumentation for any purpose, without fee, and without a written\nagreement is hereby granted, provided that the above copyright notice and\nthis paragraph and the following two paragraphs appear in all copies or in\nany new file that contains a substantial portion of this file.\n\nIN NO EVENT SHALL THE AUTHOR BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT,\nSPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS,\nARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF THE\nAUTHOR HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\nTHE AUTHOR SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED\nTO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\nPURPOSE.  THE SOFTWARE PROVIDED HEREUNDER IS ON AN \"AS IS\" BASIS, AND THE\nAUTHOR HAS NO OBLIGATIONS TO PROVIDE MAINTENANCE, SUPPORT, UPDATES,\nENHANCEMENTS, OR MODIFICATIONS."
     }
   ],
   "files": [

tests/cluecode/data/plugin_filter_clues/filtered-expected3.json

@@ -40,16 +40,18 @@
   ],
   "license_rule_references": [
     {
-      "license_expression": "pcre",
       "rule_identifier": "pcre.LICENSE",
-      "referenced_filenames": [],
+      "license_expression": "pcre",
+      "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/pcre.LICENSE",
+      "rule_relevance": 100,
+      "rule_length": 303,
       "is_license_text": true,
       "is_license_notice": false,
       "is_license_reference": false,
       "is_license_tag": false,
       "is_license_intro": false,
-      "rule_length": 303,
-      "rule_relevance": 100
+      "referenced_filenames": [],
+      "rule_text": "PCRE LICENCE\n------------\n\nPCRE is a library of functions to support regular expressions whose\nsyntax and semantics are as close as possible to those of the Perl 5\nlanguage.\n\nWritten by: Philip Hazel <ph10@cam.ac.uk>\nUniversity of Cambridge Computing Service, Cambridge, England.\nPhone: +44 1223 334714.\nCopyright (c) 1997-2001 University of Cambridge\n\nPermission is granted to anyone to use this software for any purpose on\nany computer system, and to redistribute it freely, subject to the\nfollowing restrictions:\n\n1. This software is distributed in the hope that it will be useful, but\nWITHOUT ANY WARRANTY; without even the implied warranty of\nMERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.\n\n2. The origin of this software must not be misrepresented, either by\nexplicit claim or by omission. In practice, this means that if you use\nPCRE in software which you distribute to others, commercially or\notherwise, you must put a sentence like this\n\"Regular expression support is provided by the PCRE library package,\nwhich is open source software, written by Philip Hazel, and copyright by\nthe University of Cambridge, England\"\n\nsomewhere reasonably visible in your documentation and in any relevant\nfiles or online help data or similar.\n\nA reference to the ftp site for the source, that is, to\nftp://ftp.csx.cam.ac.uk/pub/software/programming/pcre/\nshould also be given in the documentation.\n\n3. Altered versions must be plainly marked as such, and must not be\nmisrepresented as being the original software.\n\n4. If PCRE is embedded in any software that is released under the GNU\nGeneral Purpose Licence (GPL), or Lesser General Purpose Licence (LGPL),\nthen the terms of that licence shall supersede any condition above with\nwhich it is incompatible.\n\nThe documentation for PCRE, supplied in the \"doc\" directory, is\ndistributed under the same terms as the software itself.\n\nEnd PCRE LICENCE"
     }
   ],
   "files": [