ci: update to run tox for both our favoured versions of dependencies and lowest supported versions

jkowalleck · madpah · web-flow · commit 07ebedcbab15 · 2021-12-09T10:44:16.000Z
* add tox env for minimal required dependencies

Signed-off-by: Jan Kowalleck &lt;jan.kowalleck@gmail.com&gt;

* try to fix `TypedDict` typing

Signed-off-by: Jan Kowalleck &lt;jan.kowalleck@gmail.com&gt;

* fix: typing definitions to be PY 3.6 compatible

Signed-off-by: Paul Horton &lt;phorton@sonatype.com&gt;

* fix: typing definitions to be PY 3.6 compatible

Signed-off-by: Paul Horton &lt;phorton@sonatype.com&gt;

* straigtened up `sys.version_info` constraints/code-branches

Signed-off-by: Jan Kowalleck &lt;jan.kowalleck@gmail.com&gt;

* removed unused type ignores

Signed-off-by: Jan Kowalleck &lt;jan.kowalleck@gmail.com&gt;

* try to fix type variants

Signed-off-by: Jan Kowalleck &lt;jan.kowalleck@gmail.com&gt;

* try to fix type variants

Signed-off-by: Jan Kowalleck &lt;jan.kowalleck@gmail.com&gt;

* typing for py3.6

Signed-off-by: Paul Horton &lt;phorton@sonatype.com&gt;

* fixed invalid unittest

Signed-off-by: Paul Horton &lt;phorton@sonatype.com&gt;

* typing for py3.6

Signed-off-by: Jan Kowalleck &lt;jan.kowalleck@gmail.com&gt;

* mypy silence `warn_unused_ignores`

Signed-off-by: Jan Kowalleck &lt;jan.kowalleck@gmail.com&gt;

* mypy in tox for lowest version is pinned

Signed-off-by: Jan Kowalleck &lt;jan.kowalleck@gmail.com&gt;

Co-authored-by: Paul Horton &lt;phorton@sonatype.com&gt;
diff --git a/.github/workflows/poetry.yml b/.github/workflows/poetry.yml
@@ -14,10 +14,12 @@ on:
 
 env:
   REPORTS_DIR: CI_reports
+  PYTHON_VERISON_DEFAULT: "3.10"
+  POETRY_VERSION: "1.1.11"
 
 jobs:
   coding-standards:
-    name: Linting & Coding Standards
+    name: Linting & CodingStandards
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
@@ -27,25 +29,35 @@ jobs:
         # see https://github.com/actions/setup-python
         uses: actions/setup-python@v2
         with:
-          python-version: 3.9
+          python-version: ${{ env.PYTHON_VERISON_DEFAULT }}
           architecture: 'x64'
       - name: Install poetry
         # see https://github.com/marketplace/actions/setup-poetry
         uses: Gr1N/setup-poetry@v7
         with:
-          poetry-version: 1.1.8
+          poetry-version: ${{ env.POETRY_VERSION }}
       - uses: actions/cache@v2
         with:
           path: ~/.cache/pypoetry/virtualenvs
-          key: ${{ runner.os }}-poetry-${{ hashFiles('poetry.lock') }}
+          key: ${{ runner.os }}-${{ env.PYTHON_VERISON_DEFAULT }}-poetry${{ env.POETRY_VERSION }}-${{ hashFiles('poetry.lock') }}
       - name: Install dependencies
-        run: poetry install
+        run: poetry install --no-root
       - name: Run tox
         run: poetry run tox -e flake8
 
   static-code-analysis:
-    name: Static Coding Analysis
+    name: StaticCodingAnalysis (py${{ matrix.python-version}} ${{ matrix.toxenv-factor }})
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - # test with the locked dependencies
+            python-version: '3.10'
+            toxenv-factor: 'locked'
+          - # test with the lowest dependencies
+            python-version: '3.6'
+            toxenv-factor: 'lowest'
     steps:
       - name: Checkout
         # see https://github.com/actions/checkout
@@ -54,37 +66,43 @@ jobs:
         # see https://github.com/actions/setup-python
         uses: actions/setup-python@v2
         with:
-          python-version: 3.9
+          python-version: ${{ matrix.python-version }}
           architecture: 'x64'
       - name: Install poetry
         # see https://github.com/marketplace/actions/setup-poetry
         uses: Gr1N/setup-poetry@v7
         with:
-          poetry-version: 1.1.8
+          poetry-version: ${{ env.POETRY_VERSION }}
       - uses: actions/cache@v2
         with:
           path: ~/.cache/pypoetry/virtualenvs
-          key: ${{ runner.os }}-poetry-${{ hashFiles('poetry.lock') }}
+          key: ${{ runner.os }}-${{ matrix.python-version }}-poetry${{ env.POETRY_VERSION }}-${{ hashFiles('poetry.lock') }}
       - name: Install dependencies
-        run: poetry install
+        run: poetry install --no-root
       - name: Run tox
-        run: poetry run tox -e mypy
+        run: poetry run tox -e mypy-${{ matrix.toxenv-factor }}
 
   build-and-test:
-    name: Build & Test Python ${{ matrix.python-version }} on ${{ matrix.os }}
+    name: Test (${{ matrix.os }} py${{ matrix.python-version }} ${{ matrix.toxenv-factor }})
     runs-on: ${{ matrix.os }}
     env:
       REPORTS_ARTIFACT: tests-reports
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, windows-latest, macos-latest]
+        os: ['ubuntu-latest', 'windows-latest', 'macos-latest']
         python-version:
           - "3.10" # highest supported
           - "3.9"
           - "3.8"
           - "3.7"
           - "3.6" # lowest supported
+        toxenv-factor: ['locked']
+        include:
+          - # test with the lowest dependencies
+            os: 'ubuntu-latest'
+            python-version: '3.6'
+            toxenv: 'lowest'
     timeout-minutes: 30
     steps:
       - name: Disabled Git auto EOL CRLF transforms
@@ -108,17 +126,17 @@ jobs:
         # see https://github.com/marketplace/actions/setup-poetry
         uses: Gr1N/setup-poetry@v7
         with:
-          poetry-version: 1.1.11
+          poetry-version: ${{ env.POETRY_VERSION }}
       - uses: actions/cache@v2
         with:
           path: ~/.cache/pypoetry/virtualenvs
-          key: ${{ runner.os }}}-${{ matrix.python-version }}-poetry-${{ hashFiles('poetry.lock') }}
+          key: ${{ runner.os }}}-${{ matrix.python-version }}-poetry${{ env.POETRY_VERSION }}-${{ hashFiles('poetry.lock') }}
       - name: Install dependencies
-        run: poetry install
+        run: poetry install --no-root
       - name: Ensure build successful
         run: poetry build
       - name: Run tox
-        run: poetry run tox -e py -s false
+        run: poetry run tox -e py-${{ matrix.toxenv-factor }} -s false
       - name: Generate coverage reports
         run: >
           poetry run coverage report &&
diff --git a/.mypy.ini b/.mypy.ini
@@ -23,10 +23,12 @@ check_untyped_defs          = True
 disallow_untyped_decorators = True
 no_implicit_optional        = True
 warn_redundant_casts        = True
-warn_unused_ignores         = True
 warn_return_any             = True
 no_implicit_reexport        = True
 
+# needed to silence some py37|py38 differences
+warn_unused_ignores         = False
+
 [mypy-pytest.*]
 ignore_missing_imports = True
 
diff --git a/cyclonedx/model/bom.py b/cyclonedx/model/bom.py
@@ -83,15 +83,16 @@ def __repr__(self) -> str:
         return '<Tool {}:{}:{}>'.format(self._vendor, self._name, self._version)
 
 
-if sys.version_info >= (3, 8, 0):
+if sys.version_info >= (3, 8):
     from importlib.metadata import version as meta_version
 else:
-    from importlib_metadata import version as meta_version  # type: ignore
+    from importlib_metadata import version as meta_version
 
 try:
-    ThisTool = Tool(vendor='CycloneDX', name='cyclonedx-python-lib', version=meta_version('cyclonedx-python-lib'))
+    __ThisToolVersion: Optional[str] = str(meta_version('cyclonedx-python-lib'))  # type: ignore[no-untyped-call]
 except Exception:
-    ThisTool = Tool(vendor='CycloneDX', name='cyclonedx-python-lib', version='UNKNOWN')
+    __ThisToolVersion = None
+ThisTool = Tool(vendor='CycloneDX', name='cyclonedx-python-lib', version=__ThisToolVersion or 'UNKNOWN')
 
 
 class BomMetaData:
diff --git a/cyclonedx/output/json.py b/cyclonedx/output/json.py
@@ -18,7 +18,7 @@
 # Copyright (c) OWASP Foundation. All Rights Reserved.
 
 import json
-from typing import Union
+from typing import Dict, List, Union
 
 from . import BaseOutput
 from .schema import BaseSchemaVersion, SchemaVersion1Dot0, SchemaVersion1Dot1, SchemaVersion1Dot2, SchemaVersion1Dot3
@@ -47,8 +47,8 @@ def _get_json(self) -> object:
         return response
 
     def _get_component_as_dict(self, component: Component) -> object:
-        c: dict[str, Union[str, list[dict[str, str]], list[dict[str, dict[str, str]]], list[
-            dict[str, Union[str, list[dict[str, str]]]]]]] = {
+        c: Dict[str, Union[str, List[Dict[str, str]], List[Dict[str, Dict[str, str]]], List[
+            Dict[str, Union[str, List[Dict[str, str]]]]]]] = {
             "type": component.get_type().value,
             "name": component.get_name(),
             "version": component.get_version(),
@@ -59,7 +59,7 @@ def _get_component_as_dict(self, component: Component) -> object:
             c['group'] = str(component.get_namespace())
 
         if component.get_hashes():
-            hashes: list[dict[str, str]] = []
+            hashes: List[Dict[str, str]] = []
             for component_hash in component.get_hashes():
                 hashes.append({
                     "alg": component_hash.get_algorithm().value,
@@ -68,7 +68,7 @@ def _get_component_as_dict(self, component: Component) -> object:
             c['hashes'] = hashes
 
         if component.get_license():
-            licenses: list[dict[str, dict[str, str]]] = [
+            licenses: List[Dict[str, Dict[str, str]]] = [
                 {
                     "license": {
                         "name": str(component.get_license())
@@ -81,9 +81,9 @@ def _get_component_as_dict(self, component: Component) -> object:
             c['author'] = str(component.get_author())
 
         if self.component_supports_external_references() and component.get_external_references():
-            ext_references: list[dict[str, Union[str, list[dict[str, str]]]]] = []
+            ext_references: List[Dict[str, Union[str, List[Dict[str, str]]]]] = []
             for ext_ref in component.get_external_references():
-                ref: dict[str, Union[str, list[dict[str, str]]]] = {
+                ref: Dict[str, Union[str, List[Dict[str, str]]]] = {
                     "type": ext_ref.get_reference_type().value,
                     "url": ext_ref.get_url()
                 }
@@ -92,7 +92,7 @@ def _get_component_as_dict(self, component: Component) -> object:
                     ref['comment'] = str(ext_ref.get_comment())
 
                 if ext_ref.get_hashes():
-                    ref_hashes: list[dict[str, str]] = []
+                    ref_hashes: List[Dict[str, str]] = []
                     for ref_hash in ext_ref.get_hashes():
                         ref_hashes.append({
                             "alg": ref_hash.get_algorithm().value,
@@ -107,21 +107,21 @@ def _get_component_as_dict(self, component: Component) -> object:
 
     def _get_metadata_as_dict(self) -> object:
         bom_metadata = self.get_bom().get_metadata()
-        metadata: dict[str, Union[str, list[dict[str, Union[str, list[dict[str, str]]]]]]] = {
+        metadata: Dict[str, Union[str, List[Dict[str, Union[str, List[Dict[str, str]]]]]]] = {
             "timestamp": bom_metadata.get_timestamp().isoformat()
         }
 
         if self.bom_metadata_supports_tools():
-            tools: list[dict[str, Union[str, list[dict[str, str]]]]] = []
+            tools: List[Dict[str, Union[str, List[Dict[str, str]]]]] = []
             for tool in bom_metadata.get_tools():
-                tool_dict: dict[str, Union[str, list[dict[str, str]]]] = {
+                tool_dict: Dict[str, Union[str, List[Dict[str, str]]]] = {
                     "vendor": tool.get_vendor(),
                     "name": tool.get_name(),
                     "version": tool.get_version()
                 }
 
                 if len(tool.get_hashes()) > 0:
-                    hashes: list[dict[str, str]] = []
+                    hashes: List[Dict[str, str]] = []
                     for tool_hash in tool.get_hashes():
                         hashes.append({
                             "alg": tool_hash.get_algorithm().value,
diff --git a/cyclonedx/parser/environment.py b/cyclonedx/parser/environment.py
@@ -31,15 +31,14 @@
 import sys
 from pkg_resources import DistInfoDistribution  # type: ignore
 
-if sys.version_info >= (3, 8, 0):
+if sys.version_info >= (3, 8):
     from importlib.metadata import metadata
-    import email
+    from email.message import Message as _MetadataReturn
 else:
-    from importlib_metadata import metadata  # type: ignore
-    import email
+    from importlib_metadata import metadata
+    from importlib_metadata._meta import PackageMetadata as _MetadataReturn
 
 from . import BaseParser
-
 from ..model.component import Component
 
 
@@ -60,22 +59,19 @@ def __init__(self) -> None:
             c = Component(name=i.project_name, version=i.version)
 
             i_metadata = self._get_metadata_for_package(i.project_name)
-            if 'Author' in i_metadata.keys():
-                c.set_author(author=i_metadata.get('Author'))
+            if 'Author' in i_metadata:
+                c.set_author(author=i_metadata['Author'])
 
-            if 'License' in i_metadata.keys() and i_metadata.get('License') != 'UNKNOWN':
-                c.set_license(license_str=i_metadata.get('License'))
+            if 'License' in i_metadata and i_metadata['License'] != 'UNKNOWN':
+                c.set_license(license_str=i_metadata['License'])
 
-            if 'Classifier' in i_metadata.keys():
-                for classifier in i_metadata.get_all('Classifier'):
+            if 'Classifier' in i_metadata:
+                for classifier in i_metadata['Classifier']:
                     if str(classifier).startswith('License :: OSI Approved :: '):
                         c.set_license(license_str=str(classifier).replace('License :: OSI Approved :: ', '').strip())
 
             self._components.append(c)
 
     @staticmethod
-    def _get_metadata_for_package(package_name: str) -> email.message.Message:
-        if sys.version_info >= (3, 8, 0):
-            return metadata(package_name)
-        else:
-            return metadata(package_name)
+    def _get_metadata_for_package(package_name: str) -> _MetadataReturn:
+        return metadata(package_name)
diff --git a/cyclonedx/utils/conda.py b/cyclonedx/utils/conda.py
@@ -21,7 +21,7 @@
 from json import JSONDecodeError
 from typing import Optional
 
-if sys.version_info >= (3, 8, 0):
+if sys.version_info >= (3, 8):
     from typing import TypedDict
 else:
     from typing_extensions import TypedDict
diff --git a/pyproject.toml b/pyproject.toml
@@ -41,6 +41,7 @@ keywords = [
 "Bug Tracker" = "https://github.com/CycloneDX/cyclonedx-python-lib/issues"
 
 [tool.poetry.dependencies]
+# keep `requirements.lowest.txt` file in sync
 python = "^3.6"
 packageurl-python = "^0.9.4"
 requirements_parser = "^0.2.0"
diff --git a/requirements.lowest.txt b/requirements.lowest.txt
@@ -0,0 +1,11 @@
+# exactly pinned dependencies to the lowest version regardless of python_version
+# see pyptoject file for ranges
+
+packageurl-python == 0.9.4
+requirements_parser == 0.2.0
+setuptools == 50.3.2
+importlib-metadata == 4.8.1 # ; python_version < '3.8'
+toml == 0.10.2
+typing-extensions == 3.10.0 # ; python_version < '3.8'
+types-setuptools == 57.4.2
+types-toml == 0.10.1
diff --git a/tests/test_parser_environment.py b/tests/test_parser_environment.py
@@ -38,4 +38,4 @@ def test_simple(self) -> None:
         # We can only be sure that tox is in the environment, for example as we use tox to run tests
         c_tox: Component = [x for x in parser.get_components() if x.get_name() == 'tox'][0]
         self.assertIsNotNone(c_tox.get_license())
-        self.assertEqual('MIT License', c_tox.get_license())
+        self.assertEqual('MIT', c_tox.get_license())
diff --git a/tox.ini b/tox.ini
@@ -7,8 +7,8 @@
 minversion = 3.10
 envlist =
     flake8
-    mypy
-    py{310,39,38,37,36}
+    mypy-{locked,lowest}
+    py{310,39,38,37,36}-{locked,lowest}
 isolated_build = True
 skip_missing_interpreters = True
 usedevelop = False
@@ -18,17 +18,21 @@ download = False
 # settings in this category apply to all other testenv, if not overwritten
 skip_install = True
 whitelist_externals = poetry
-deps = poetry
+deps =
+    poetry
 commands_pre =
     {envpython} --version
     poetry install -v
+    lowest: poetry run pip install -U -r requirements.lowest.txt
+    poetry run pip freeze
 commands =
-    poetry run coverage run --source=cyclonedx -m unittest discover -s tests
+    poetry run coverage run --source=cyclonedx -m unittest discover -s tests -v
 
-[testenv:mypy]
+[testenv:mypy{,-locked,-lowest}]
 commands =
-    poetry run mypy
-    # mypy config is on own file: `.mypy.ini`
+    # mypy config is in own file: `.mypy.ini`
+    !lowest: poetry run mypy
+     lowest: poetry run mypy --python-version=3.6
 
 [testenv:flake8]
 commands =
