Add support for setup.py

Signed-off-by: Tushar Goel <tushar.goel.dav@gmail.com>

Author: TG1999

src/_packagedcode/pypi.py

@@ -659,6 +659,22 @@ class ResolvedPurl(NamedTuple):
     is_resolved: bool
 
 
+def create_dependency_for_python_requires(python_requires_specifier):
+    """
+    Return a mock python DependentPackage created from a ``python_requires_specifier``.
+    """
+    purl = PackageURL(type="generic", name="python")
+    resolved_purl = get_resolved_purl(purl=purl, specifiers=SpecifierSet(python_requires_specifier))
+    return models.DependentPackage(
+        purl=str(resolved_purl.purl),
+        scope="python",
+        is_runtime=True,
+        is_optional=False,
+        is_resolved=resolved_purl.is_resolved,
+        extracted_requirement=f"python_requires{python_requires_specifier}",
+    )
+
+
 class BaseDependencyFileHandler(BasePypiHandler):
     """
     Base class for a dependency files parsed with the same library
@@ -717,19 +733,8 @@ def parse(cls, location):
                         dependent_packages.extend(cls.parse_reqs(reqs, scope))
                         continue
                     python_requires_specifier = section[sub_section]
-                    purl = PackageURL(
-                        type="generic",
-                        name="python",
-                    )
-                    resolved_purl = get_resolved_purl(purl=purl, specifiers=SpecifierSet(python_requires_specifier))
-                    dependent_packages.append(models.DependentPackage(
-                    purl=str(resolved_purl.purl),
-                    scope=scope,
-                    is_runtime=True,
-                    is_optional=False,
-                    is_resolved=resolved_purl.is_resolved,
-                    extracted_requirement=f"python_requires{python_requires_specifier}",
-                    ))
+                    pd = create_dependency_for_python_requires(python_requires_specifier)
+                    dependent_packages.append(pd)
 
             if section.name == "options.extras_require":
                 for sub_section in section:

src/python_inspector/dependencies.py

@@ -15,6 +15,8 @@
 
 from _packagedcode import models
 from _packagedcode.pypi import PipRequirementsFileHandler
+from _packagedcode.pypi import PythonSetupPyHandler
+from python_inspector.resolution import get_requirements_from_distribution
 
 """
 Utilities to resolve dependencies .
@@ -23,7 +25,7 @@
 TRACE = False
 
 
-def get_dependencies_from_requirements(requirements_file="requirements.txt", *args, **kwargs):
+def get_dependencies_from_requirements(requirements_file="requirements.txt"):
     """
     Yield DependentPackage for each requirement in a `requirement`
     file.
@@ -38,7 +40,7 @@ def get_dependencies_from_requirements(requirements_file="requirements.txt", *ar
             yield dependent_package
 
 
-def get_extra_data_from_requirements(requirements_file="requirements.txt", *args, **kwargs):
+def get_extra_data_from_requirements(requirements_file="requirements.txt"):
     """
     Yield extra_data for each requirement in a `requirement`
     file.
@@ -47,7 +49,7 @@ def get_extra_data_from_requirements(requirements_file="requirements.txt", *args
         yield package_data.extra_data
 
 
-def get_dependency(specifier, *args, **kwargs):
+def get_dependency(specifier):
     """
     Return a DependentPackage given a requirement ``specifier`` string.
 

src/python_inspector/resolution.py

@@ -15,6 +15,7 @@
 from typing import List
 from typing import NamedTuple
 from typing import Sequence
+from typing import Tuple
 from typing import Union
 from zipfile import ZipFile
 
@@ -88,17 +89,20 @@ def get_response(url: str) -> Dict:
 
 
 def get_requirements_from_distribution(
-    handler: BasePypiHandler, location: str
+    handler: BasePypiHandler,
+    location: str,
 ) -> List[Requirement]:
     """
     Return a list of requirements from a source distribution or wheel at
     ``location`` using the provided ``handler`` DatafileHandler for parsing.
     """
     if not os.path.exists(location):
         return []
-    deps = list(handler.parse(location))
-    assert len(deps) == 1
-    return list(get_requirements_from_dependencies(dependencies=deps[0].dependencies))
+    deps = []
+    for package_data in handler.parse(location):
+        dependencies = package_data.dependencies
+        deps.extend(get_requirements_from_dependencies(dependencies=dependencies))
+    return deps
 
 
 def get_environment_marker_from_environment(environment):
@@ -200,7 +204,9 @@ def fetch_and_extract_sdist(
     return os.path.join(utils_pypi.CACHE_THIRDPARTY_DIR, "extracted_sdists", sdist_file, sdist_file)
 
 
-def get_requirements_from_dependencies(dependencies: List[DependentPackage]) -> List[Requirement]:
+def get_requirements_from_dependencies(
+    dependencies: List[DependentPackage], scopes: Tuple[str] = ("install",)
+) -> List[Requirement]:
     """
     Generate parsed requirements for the given ``dependencies``.
     """
@@ -209,7 +215,7 @@ def get_requirements_from_dependencies(dependencies: List[DependentPackage]) ->
             continue
 
         # TODO: consider other scopes and using the is_runtime flag
-        if dep.scope != "install":
+        if dep.scope not in scopes:
             continue
 
         # FIXME We are skipping editable requirements
@@ -284,12 +290,16 @@ def get_versions_for_package_from_repo(
             if wheels:
                 valid_wheel_present = False
                 for wheel in wheels:
-                    if utils_pypi.valid_distribution(wheel, python_version):
+                    if utils_pypi.valid_python_version(
+                        python_requires=wheel.python_requires, python_version=python_version
+                    ):
                         valid_wheel_present = True
                 if valid_wheel_present:
                     versions.append(version)
             if package.sdist:
-                if utils_pypi.valid_distribution(package.sdist, python_version):
+                if utils_pypi.valid_python_version(
+                    python_requires=package.sdist.python_requires, python_version=python_version
+                ):
                     versions.append(version)
         return versions
 

src/python_inspector/resolve_cli.py

@@ -19,13 +19,14 @@
 from tinynetrc import Netrc
 
 from _packagedcode.models import DependentPackage
-from _packagedcode.pypi import PipRequirementsFileHandler
+from _packagedcode.pypi import PythonSetupPyHandler
 from _packagedcode.pypi import can_process_dependent_package
 from python_inspector import dependencies
 from python_inspector import utils
 from python_inspector import utils_pypi
 from python_inspector.cli_utils import FileOptionType
 from python_inspector.resolution import get_environment_marker_from_environment
+from python_inspector.resolution import get_python_version_from_env_tag
 from python_inspector.resolution import get_resolved_dependencies
 
 TRACE = False
@@ -50,13 +51,14 @@
     "This option can be used multiple times.",
 )
 @click.option(
-    "-n",
-    "--netrc",
-    "netrc_file",
+    "-s",
+    "--setup-py",
+    "setup_py_file",
     type=click.Path(exists=True, readable=True, path_type=str, dir_okay=False),
-    metavar="NETRC-FILE",
+    metavar="SETUP-PY-FILE",
     required=False,
-    help="Netrc file to use for authentication. ",
+    help="Path to setuptools setup.py file listing dependencies and metadata. "
+    "This option can be used multiple times.",
 )
 @click.option(
     "--spec",
@@ -117,9 +119,20 @@
     help="Write output as pretty-printed JSON to FILE as a tree in the style of pipdeptree. "
     "Use the special '-' file name to print results on screen/stdout.",
 )
+@click.option(
+    "-n",
+    "--netrc",
+    "netrc_file",
+    type=click.Path(exists=True, readable=True, path_type=str, dir_okay=False),
+    metavar="NETRC-FILE",
+    hidden=True,
+    required=False,
+    help="Netrc file to use for authentication. ",
+)
 @click.option(
     "--max-rounds",
     "max_rounds",
+    hidden=True,
     type=int,
     default=200000,
     help="Increase the max rounds whenever the resolution is too deep",
@@ -146,21 +159,23 @@
 def resolve_dependencies(
     ctx,
     requirement_files,
-    netrc_file,
+    setup_py_file,
     specifiers,
     python_version,
     operating_system,
     index_urls,
     json_output,
     pdt_output,
+    netrc_file,
     max_rounds,
     use_cached_index=False,
     use_pypi_json_api=False,
     verbose=TRACE,
 ):
     """
-    Resolve the dependencies of the packages listed in REQUIREMENT-FILE(s) file
-    and SPECIFIER(s) and save the results as JSON to FILE.
+    Resolve the dependencies for the package requirements listed in one or
+    more REQUIREMENT-FILE file, one or more SPECIFIER and one setuptools
+    SETUP-PY-FILE file and save the results as JSON to FILE.
 
     Resolve the dependencies for the requested ``--python-version`` PYVER and
     ``--operating_system`` OS combination defaulting Python version 3.8 and
@@ -221,6 +236,28 @@ def resolve_dependencies(
         dep = dependencies.get_dependency(specifier=specifier)
         direct_dependencies.append(dep)
 
+    if setup_py_file:
+        package_data = list(PythonSetupPyHandler.parse(location=setup_py_file))
+        assert len(package_data) == 1
+        package_data = package_data[0]
+        # validate if python require matches our current python version
+        python_requires = package_data.extra_data.get("python_requires")
+        if not utils_pypi.valid_python_version(
+            python_version=get_python_version_from_env_tag(python_version),
+            python_requires=python_requires,
+        ):
+            click.secho(
+                f"Python version {python_version} is not compatible with setup.py {setup_py_file} "
+                f"python_requires {python_requires}",
+                err=True,
+            )
+            ctx.exit(1)
+
+        for dep in package_data.dependencies:
+            # TODO : we need to handle to all the scopes
+            if dep.scope == "install":
+                direct_dependencies.append(dep)
+
     if not direct_dependencies:
         click.secho("Error: no requirements requested.")
         ctx.exit(1)

src/python_inspector/utils_pypi.py

@@ -98,14 +98,15 @@
 TRACE_ULTRA_DEEP = False
 
 # Supported environments
-PYTHON_VERSIONS = "36", "37", "38", "39", "310"
+PYTHON_VERSIONS = "36", "37", "38", "39", "310", "27"
 
 PYTHON_DOT_VERSIONS_BY_VER = {
     "36": "3.6",
     "37": "3.7",
     "38": "3.8",
     "39": "3.9",
     "310": "3.10",
+    "27": "2.7",
 }
 
 
@@ -122,6 +123,7 @@ def get_python_dot_version(version):
     "38": ["cp38", "cp38m", "abi3"],
     "39": ["cp39", "cp39m", "abi3"],
     "310": ["cp310", "cp310m", "abi3"],
+    "27": ["cp27", "cp27m"],
 }
 
 PLATFORMS_BY_OS = {
@@ -256,7 +258,9 @@ def get_valid_sdist(repo, name, version, python_version=DEFAULT_PYTHON_VERSION):
         if TRACE_DEEP:
             print(f"    get_valid_sdist: No sdist for {name}=={version}")
         return
-    if not valid_distribution(sdist, python_version):
+    if not valid_python_version(
+        python_requires=sdist.python_requires, python_version=python_version
+    ):
         return
     if TRACE_DEEP:
         print(f"    get_valid_sdist: Getting sdist from index (or cache): {sdist.download_url}")
@@ -285,7 +289,9 @@ def get_supported_and_valid_wheels(
         return []
     wheels = []
     for wheel in supported_wheels:
-        if not valid_distribution(wheel, python_version):
+        if not valid_python_version(
+            python_requires=wheel.python_requires, python_version=python_version
+        ):
             continue
         if TRACE_DEEP:
             print(
@@ -296,13 +302,13 @@ def get_supported_and_valid_wheels(
     return wheels
 
 
-def valid_distribution(distribution, python_version):
+def valid_python_version(python_version, python_requires):
     """
-    Return True if distribution is a valid distribution for the given Python version.
+    Return True if ``python_version`` is in the ``python_requires``.
     """
-    if not distribution.python_requires:
+    if not python_requires:
         return True
-    return python_version in SpecifierSet(distribution.python_requires)
+    return python_version in SpecifierSet(python_requires)
 
 
 def download_sdist(

tests/data/setup/simple-setup.py

@@ -0,0 +1,14 @@
+from setuptools import find_packages
+from setuptools import setup
+
+setup(
+    name="Example-App",
+    description="A synthetic test case for OSS Review Toolkit",
+    version="2.4.0",
+    url="https://example.org/app",
+    license="MIT License",
+    classifiers=["License :: OSI Approved :: MIT License", "Programming Language :: Python :: 2"],
+    python_requires=">=2, <3",
+    install_requires=["Flask>=0.12, <1.1"],
+    packages=find_packages(),
+)

tests/test_requirement_parsing.py

@@ -1,12 +1,23 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# ScanCode is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/nexB/python-inspector for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
 import os
 
 from python_inspector.dependencies import get_extra_data_from_requirements
 
 BASE_DIR = os.path.dirname(os.path.abspath(__file__))
-REQ_FILE = os.path.join(BASE_DIR, "data", "requirements-test.txt")
 
 
 def test_get_extra_data_from_requirements():
+    req_file = os.path.join(BASE_DIR, "data", "requirements-test.txt")
     expected = [
         {
             "extra_index_urls": [
@@ -16,5 +27,5 @@ def test_get_extra_data_from_requirements():
             ]
         }
     ]
-    result = list(get_extra_data_from_requirements(REQ_FILE))
+    result = list(get_extra_data_from_requirements(req_file))
     assert expected == result

tests/test_resolution.py

@@ -145,7 +145,7 @@ def test_without_supported_wheels():
         "pkg:pypi/hyperlink@21.0.0",
         "pkg:pypi/idna@3.3",
         "pkg:pypi/pycparser@2.21",
-        "pkg:pypi/setuptools@65.0.1",
+        "pkg:pypi/setuptools@65.1.0",
         "pkg:pypi/txaio@22.2.1",
     ]