✨ Licence compliance check (#45)

* creating a way to identify equivalent strings

* Added unit tests

* modified spdx report to use licence lookup

* modified configuration to list accepted licences

* checking licence compliance of a package and its dependencies

* copyright and news file

* making code climate happy

* making code climate even more happy and hopefully for good

* formatting improvement

* try forcing publishing artifacts even if check fails

* Update mbed_tools_ci_scripts/utils/third_party_licences.py

Co-Authored-By: Rob Walton <rob.walton@arm.com>

* changes following review comments

* Updated the PR to handle more licence descriptor cases.

* Adding a way to use a regex to defined allowed licences e.g. `BSD* `

* licence expression understanding improvement

* fixing problems with comma in the licence description

* fix regex

* fix test

* Better handled licences and put in place a way to mark packages as checked with regards to licensing

* listing the packages for which the licence has been checked

Co-authored-by: Rob Walton <rob.walton@arm.com>

Author: acabarbaye

azure-pipelines/build-release.yml

@@ -153,6 +153,7 @@ stages:
           - publish: $(temp_spdx_reports_path)
             artifact: SPDX
             displayName: 'Publish SPDX reports'
+            condition: always()
 
   # Collect test and build stages together before the release stages to provide a pass/fail point for the status badge.
   - stage: CiCheckpoint

mbed_tools_ci_scripts/report_third_party_ip.py

@@ -16,15 +16,15 @@
 import argparse
 import logging
 from pathlib import Path
-
+from typing import Any
 from mbed_tools_ci_scripts.spdx_report.spdx_project import SpdxProject
 from mbed_tools_ci_scripts.utils.logging import set_log_level, log_exception
 from mbed_tools_ci_scripts.utils.package_helpers import CurrentProjectMetadataParser, generate_package_info
 
 logger = logging.getLogger(__name__)
 
 
-def generate_spdx_reports(output_directory: Path) -> None:
+def generate_spdx_reports(output_directory: Path) -> SpdxProject:
     """Generates all the SPDX reports for the current project."""
     logger.info("Generating package information.")
     try:
@@ -34,14 +34,21 @@ def generate_spdx_reports(output_directory: Path) -> None:
         log_exception(logger, e)
 
     logger.info("Generating SPDX report.")
-    SpdxProject(CurrentProjectMetadataParser()).generate_tag_value_files(output_directory)
+    project = SpdxProject(CurrentProjectMetadataParser())
+    project.generate_tag_value_files(output_directory)
+    return project
 
 
 def main() -> int:
     """Script CLI."""
     parser = argparse.ArgumentParser(description="Generate licence and third-party IP reports.")
+
+    def convert_to_path(arg: Any) -> Path:
+        """Converts argument to a path."""
+        return Path(arg)
+
     parser.add_argument(
-        "-o", "--output-dir", help="Output directory where the files are generated", required=True,
+        "-o", "--output-dir", help="Output directory where the files are generated", required=True, type=convert_to_path
     )
 
     parser.add_argument(
@@ -51,7 +58,8 @@ def main() -> int:
     set_log_level(args.verbose)
 
     try:
-        generate_spdx_reports(Path(args.output_dir))
+        project = generate_spdx_reports(args.output_dir)
+        project.check_licence_compliance()
         return 0
     except Exception as e:
         log_exception(logger, e)

mbed_tools_ci_scripts/spdx_report/spdx_document.py

@@ -177,6 +177,11 @@ def external_refs(self) -> List[DependencySpdxDocumentRef]:
         """
         return self._other_document_references
 
+    @external_refs.setter
+    def external_refs(self, external_refs: List[DependencySpdxDocumentRef]) -> None:
+        """Sets the document external references."""
+        self._other_document_references = external_refs
+
     def generate_spdx_package(self) -> SpdxPackage:
         """Generates the SPDX package for this package.
 

mbed_tools_ci_scripts/spdx_report/spdx_file.py

@@ -17,6 +17,7 @@
 )
 from mbed_tools_ci_scripts.utils.definitions import UNKNOWN
 from mbed_tools_ci_scripts.utils.hash_helpers import generate_uuid_based_on_str, determine_sha1_hash_of_file
+from mbed_tools_ci_scripts.utils.third_party_licences import cleanse_licence_expression
 
 
 class SpdxFile:
@@ -88,7 +89,7 @@ def licence(self) -> str:
             file's licence
         """
         file_licence = determine_file_licence(self.path)
-        return file_licence if file_licence else self._package_licence
+        return cleanse_licence_expression(file_licence) if file_licence else self._package_licence
 
     @property
     def copyright(self) -> Optional[str]:

mbed_tools_ci_scripts/spdx_report/spdx_helpers.py

@@ -17,13 +17,14 @@
 
 import logging
 import toml
-from license_expression import Licensing
 from pathlib import Path
 from spdx.utils import SPDXNone, UnKnown
-from typing import Union, Optional, Iterator, List
+from typing import Union, Optional, Iterator, Iterable, Any
 
-from mbed_tools_ci_scripts.utils.filesystem_helpers import scan_file_for_pattern, should_exclude_path, list_all_files
+from mbed_tools_ci_scripts.utils.configuration import ConfigurationVariable, configuration
 from mbed_tools_ci_scripts.utils.definitions import UNKNOWN
+from mbed_tools_ci_scripts.utils.filesystem_helpers import scan_file_for_pattern, should_exclude_path, list_all_files
+from mbed_tools_ci_scripts.utils.third_party_licences import simplify_licence_expression
 
 logger = logging.getLogger(__name__)
 
@@ -56,7 +57,7 @@ def determine_file_licence(path: Path) -> Optional[str]:
         if not match:
             return None
         licence = match.group(1).strip()
-        return str(Licensing().parse(licence).simplify())
+        return simplify_licence_expression(licence)
     except Exception as e:
         logger.error(f"Could not determine the licence of file [{path}] from identifier '{licence}'. Reason: {e}.")
         return None
@@ -105,7 +106,21 @@ def ignore_path(p: Path) -> bool:
     return list_all_files(project_root, ignore_path)
 
 
-def determine_licence_compound(main_licence: str, additional_licences: List[str]) -> str:
-    """Determines the overall licence based on main licence and additional licences."""
-    overall_licence = f"({main_licence}) AND ({') AND ('.join(additional_licences)})"
-    return str(Licensing().parse(overall_licence).simplify())
+def determine_checked_packages_from_string(checked_packages: Any) -> Iterable[Any]:
+    """Determines the list of packages for which the licence has been checked."""
+    if isinstance(checked_packages, str):
+        checked_packages = checked_packages.split(", ")
+    if isinstance(checked_packages, (list, dict, tuple, set)):
+        yield from checked_packages
+
+
+def get_packages_with_checked_licence() -> Iterable[str]:
+    """Determines the list of packages for which the licence has been checked from configuration."""
+    yield from determine_checked_packages_from_string(
+        configuration.get_value(ConfigurationVariable.PACKAGES_WITH_CHECKED_LICENCE)
+    )
+
+
+def is_package_licence_checked(package_name: str) -> bool:
+    """States whether the licence of a package has been checked and hence, that its licence is compliant."""
+    return package_name.strip() in get_packages_with_checked_licence()

mbed_tools_ci_scripts/spdx_report/spdx_package.py

@@ -4,6 +4,7 @@
 #
 """Definition of an SPDX Package."""
 
+from dataclasses import dataclass
 from pathlib import Path
 from spdx.checksum import Algorithm
 from spdx.creationinfo import Person
@@ -16,11 +17,15 @@
 from mbed_tools_ci_scripts.spdx_report.spdx_helpers import (
     determine_spdx_value,
     list_project_files_for_licensing,
-    determine_licence_compound,
 )
 from mbed_tools_ci_scripts.utils.definitions import UNKNOWN
 from mbed_tools_ci_scripts.utils.package_helpers import PackageMetadata
-from dataclasses import dataclass
+from mbed_tools_ci_scripts.utils.third_party_licences import (
+    UNKNOWN_LICENCE,
+    cleanse_licence_expression,
+    is_licence_accepted,
+    determine_licence_compound,
+)
 
 
 @dataclass(frozen=True, order=True)
@@ -58,6 +63,7 @@ def __init__(self, package_info: PackageInfo, is_dependency: bool = False,) -> N
         self._package_info = package_info
         self._file_list: Optional[List[Path]] = None
         self._actual_licence: Optional[str] = None
+        self._main_licence: Optional[str] = None
 
     @property
     def files(self) -> Optional[List[Path]]:
@@ -108,7 +114,17 @@ def main_licence(self) -> str:
         Returns:
             project's licence
         """
-        return self._package_info.metadata.licence
+        if not self._main_licence:
+            package_licence = self._package_info.metadata.licence
+            self._main_licence = (
+                cleanse_licence_expression(package_licence) if package_licence else UNKNOWN_LICENCE.identifier
+            )
+        return self._main_licence
+
+    @property
+    def is_main_licence_accepted(self) -> bool:
+        """States whether the main licence of the package is part of the accepted licence list."""
+        return is_licence_accepted(self.main_licence)
 
     @property
     def licence(self) -> str:
@@ -126,6 +142,11 @@ def licence(self) -> str:
             )
         return self._actual_licence
 
+    @property
+    def is_licence_accepted(self) -> bool:
+        """States whether the actual package's licence of the package is part of the accepted licence list."""
+        return is_licence_accepted(self.licence)
+
     @property
     def author(self) -> str:
         """Gets the document's author.
@@ -168,7 +189,7 @@ def get_spdx_files(self) -> Optional[List[SpdxFile]]:
         Returns:
             list of file descriptions or None if a dependency.
         """
-        if not self.files or len(self.files) == 0:
+        if not self.files:
             return None
         return [SpdxFile(p, self._package_info.root_dir, self.main_licence) for p in self.files]
 
@@ -197,21 +218,22 @@ def generate_spdx_package(self) -> Package:
         Returns:
             the corresponding package
         """
-        package = Package()
+        package = Package(
+            name=determine_spdx_value(self.name),
+            spdx_id=f"SPDXRef-{self.id}",
+            download_location=determine_spdx_value(None),
+            version=determine_spdx_value(self.version),
+            file_name=determine_spdx_value(self.name),
+            supplier=None,
+            originator=Person(determine_spdx_value(self.author), determine_spdx_value(self.author_email)),
+        )
         package.check_sum = Algorithm("SHA1", str(NoAssert()))
         package.cr_text = NoAssert()
-        package.name = determine_spdx_value(self.name)
-        package.version = determine_spdx_value(self.version)
-        package.file_name = determine_spdx_value(self.name)
-        package.download_location = determine_spdx_value(None)
         package.homepage = determine_spdx_value(self.url)
-        package.originator = Person(determine_spdx_value(self.author), determine_spdx_value(self.author_email))
         package.license_declared = License.from_identifier(str(determine_spdx_value(self.main_licence)))
         package.conc_lics = License.from_identifier(str(determine_spdx_value(self.licence)))
         package.summary = determine_spdx_value(self.description)
         package.description = NoAssert()
-        package.spdx_id = f"SPDXRef-{self.id}"
-
         files = self.get_spdx_files()
         if files:
             package.files_analyzed = True
@@ -223,7 +245,7 @@ def generate_spdx_package(self) -> Package:
         else:
             # Has to generate a dummy file because of the following rule in SDK:
             # - Package must have at least one file
-            dummy_file = SpdxFile(Path(UNKNOWN), self._package_info.root_dir, self.licence)
+            dummy_file = SpdxFile(Path(UNKNOWN), self._package_info.root_dir, self.main_licence)
             package.verif_code = NoAssert()
             package.add_file(dummy_file.generate_spdx_file())
             package.add_lics_from_file(License.from_identifier(str(determine_spdx_value(dummy_file.licence))))

mbed_tools_ci_scripts/spdx_report/spdx_project.py

@@ -5,12 +5,15 @@
 """Definition of an SPDX report for a Python project."""
 
 from pathlib import Path
+import os
 from spdx.writers.tagvalue import write_document
+from typing import Optional, List, cast, Tuple, Dict
 
 from mbed_tools_ci_scripts.spdx_report.spdx_dependency import DependencySpdxDocumentRef
 from mbed_tools_ci_scripts.spdx_report.spdx_document import SpdxDocument
 from mbed_tools_ci_scripts.utils.hash_helpers import determine_sha1_hash_of_file
 from mbed_tools_ci_scripts.utils.package_helpers import ProjectMetadataParser
+from mbed_tools_ci_scripts.spdx_report.spdx_helpers import is_package_licence_checked
 
 
 class SpdxProject:
@@ -23,6 +26,30 @@ class SpdxProject:
     def __init__(self, parser: ProjectMetadataParser) -> None:
         """Constructor."""
         self._parser = parser
+        self._main_document: Optional[SpdxDocument] = None
+        self._dependency_documents: Optional[List[SpdxDocument]] = None
+
+    def _generate_documents(self) -> None:
+        if self._main_document:
+            return
+        self._dependency_documents = list()
+        project_metadata = self._parser.project_metadata
+        dependencies = project_metadata.dependencies_metadata
+        for dependency in dependencies:
+            self._dependency_documents.append(SpdxDocument(dependency, is_dependency=True))
+        self._main_document = SpdxDocument(package_metadata=project_metadata.project_metadata)
+
+    @property
+    def main_document(self) -> SpdxDocument:
+        """Gets project's main SPDX document."""
+        self._generate_documents()
+        return cast(SpdxDocument, self._main_document)
+
+    @property
+    def dependency_documents(self) -> List[SpdxDocument]:
+        """Gets the list of project's dependencies SPDX documents."""
+        self._generate_documents()
+        return self._dependency_documents if self._dependency_documents else list()
 
     @staticmethod
     def generate_tag_value_file(dir: Path, spdx_doc: SpdxDocument, filename: str = "LICENSE.spdx") -> str:
@@ -63,19 +90,58 @@ def generate_tag_value_files(self, dir: Path) -> None:
         if not dir.is_dir():
             raise NotADirectoryError(str(dir))
 
-        project_metadata = self._parser.project_metadata
-        externalRefs = []
-        dependencies = project_metadata.dependencies_metadata
-        for dependency in dependencies:
-            spdx_dependency = SpdxDocument(dependency, is_dependency=True)
+        externalRefs = list()
+        for spdx_dependency in self.dependency_documents:
             file_name = f"{spdx_dependency.name}.spdx"
             checksum = SpdxProject.generate_tag_value_file(dir, spdx_dependency, file_name)
             externalRefs.append(
                 DependencySpdxDocumentRef(
                     name=spdx_dependency.document_name, namespace=spdx_dependency.document_namespace, checksum=checksum,
                 )
             )
-        main_document = SpdxDocument(
-            package_metadata=project_metadata.project_metadata, other_document_refs=externalRefs,
+        self.main_document.external_refs = externalRefs
+        SpdxProject.generate_tag_value_file(dir, self.main_document, f"{self.main_document.name}.spdx")
+
+    @staticmethod
+    def _check_package_licence(package_document: SpdxDocument) -> Tuple[bool, bool, str, str, str]:
+        package = package_document.generate_spdx_package()
+        return (
+            package.is_main_licence_accepted,
+            package.is_licence_accepted,
+            package.name,
+            package.main_licence,
+            package.licence,
         )
-        SpdxProject.generate_tag_value_file(dir, main_document, f"{main_document.name}.spdx")
+
+    def _report_issues(self, issues: Dict[str, str]) -> None:
+        if issues:
+            raise ValueError(
+                f",{os.linesep}".join(
+                    [
+                        f"Package [{package_name}] has a non-compliant licence ({package_licence}) for this project"
+                        for package_name, package_licence in issues.items()
+                    ]
+                )
+            )
+
+    def _check_one_licence_compliance(self, spdx_document: SpdxDocument, issues: Dict[str, str]) -> None:
+        main_valid, actual_valid, name, main_licence, actual_licence = SpdxProject._check_package_licence(spdx_document)
+        if not ((main_valid and actual_valid) or is_package_licence_checked(name)):
+            issues[name] = actual_licence if main_valid else main_licence
+
+    def _check_package_dependencies_licence_compliance(self, issues: Dict[str, str]) -> None:
+        for dependency in self.dependency_documents:
+            self._check_one_licence_compliance(dependency, issues)
+
+    def _check_package_licence_compliance(self, issues: Dict[str, str]) -> None:
+        self._check_one_licence_compliance(self.main_document, issues)
+
+    def check_licence_compliance(self) -> None:
+        """Checks whether the licences of the package as well as all its dependencies are compliant.
+
+        By compliant, it is meant that all the licences are in the list of accepted licences set for the given project.
+        """
+        issues: Dict[str, str] = dict()
+        self._check_package_licence_compliance(issues)
+        self._check_package_dependencies_licence_compliance(issues)
+        self._report_issues(issues)

mbed_tools_ci_scripts/utils/configuration.py

@@ -46,6 +46,8 @@ class ConfigurationVariable(enum.Enum):
     IGNORE_PYPI_TEST_UPLOAD = 25
     FILE_LICENCE_IDENTIFIER = 26
     COPYRIGHT_START_DATE = 27
+    ACCEPTED_THIRD_PARTY_LICENCES = 28
+    PACKAGES_WITH_CHECKED_LICENCE = 29
 
     @staticmethod
     def choices() -> List[str]:
@@ -146,6 +148,8 @@ class StaticConfig(GenericConfig):
     ORGANISATION_EMAIL = "support@mbed.com"
     FILE_LICENCE_IDENTIFIER = "Apache-2.0"
     COPYRIGHT_START_DATE = 2020
+    ACCEPTED_THIRD_PARTY_LICENCES = ["Apache-2.0", "BSD*", "JSON", "MIT", "Python-2.0", "PSF-2.0", "MPL-2.0"]
+    PACKAGES_WITH_CHECKED_LICENCE: List[str] = []
 
     def _fetch_value(self, key: str) -> Any:
         try: