Address review comment

Signed-off-by: Tushar Goel <tushar.goel.dav@gmail.com>

Author: TG1999

src/_packagedcode/pypi.py

@@ -14,6 +14,7 @@
 import os
 import re
 import sys
+from typing import NamedTuple
 import zipfile
 from configparser import ConfigParser
 from pathlib import Path
@@ -23,6 +24,7 @@
 import pip_requirements_parser
 import pkginfo2
 from commoncode import fileutils
+from packaging.specifiers import SpecifierSet
 from packageurl import PackageURL
 from packaging import markers
 from packaging.requirements import Requirement
@@ -453,8 +455,9 @@ def parse_metadata(location, datasource_id, package_type):
         type=package_type,
         primary_language='Python',
         name=name,
-        version=version, #TODO: https://github.com/nexB/scancode-toolkit/issues/3014
-        description=get_description(meta, str(location)),
+        version=version,
+        description=get_description( metainfo= meta, location= str(location)),
+        #TODO: https://github.com/nexB/scancode-toolkit/issues/3014
         declared_license=get_declared_license(meta),
         keywords=get_keywords(meta),
         parties=get_parties(meta),
@@ -646,6 +649,14 @@ def parse(cls, location):
         )
 
 
+class ResolvedPurl(NamedTuple):
+    """
+    A resolved PURL
+    """
+    purl: PackageURL
+    is_resolved: bool
+
+
 class BaseDependencyFileHandler(BasePypiHandler):
     """
     Base class for a dependency files parsed with the same library
@@ -690,8 +701,35 @@ def parse(cls, location):
             parser.read_file(f)
         for section in parser.values():
             if section.name == 'options':
-                reqs = list(get_requirement_from_section(section=section, sub_section="install_requires"))
-                dependent_packages.extend(cls.parse_reqs(reqs, "install"))
+                scope_by_sub_section = {
+                    "install_requires": "install",
+                    "tests_require": "test",
+                    "setup_requires": "setup",
+                    "python_requires": "python",
+                }
+                for sub_section in scope_by_sub_section:
+                    if sub_section not in section:
+                        continue
+                    scope = scope_by_sub_section[sub_section]
+                    if scope != "python":
+                        reqs = list(get_requirement_from_section(section=section, sub_section=sub_section))
+                        dependent_packages.extend(cls.parse_reqs(reqs, scope))
+                        continue
+                    python_requires = section[sub_section]
+                    purl = PackageURL(
+                        name="python",
+                        type="generic"
+                    )
+                    resolved_purl = is_purl_resolved(purl = purl, specifiers= SpecifierSet(python_requires))
+                    dependent_packages.append(models.DependentPackage(
+                    purl=str(resolved_purl.purl),
+                    scope=scope,
+                    is_runtime=True,
+                    is_optional=False,
+                    is_resolved=resolved_purl.is_resolved,
+                    extracted_requirement=f"python_requires{python_requires}",
+                    ))
+
             if section.name == "options.extras_require":
                 for sub_section in section:
                     reqs = list(get_requirement_from_section(section=section, sub_section=sub_section))
@@ -742,28 +780,39 @@ def parse_reqs(cls, reqs, scope):
         """ 
         dependent_packages = []
         for req in reqs:
-            is_resolved = False
             req_parsed = packaging.requirements.Requirement(str(req))
             name = canonicalize_name(req_parsed.name)
             purl = PackageURL(type="pypi", name=name)
             specifiers = req_parsed.specifier._specs
-            if len(specifiers) == 1:
-                specifier = list(specifiers)[0]
-                if specifier.operator in ('==', '==='):
-                    is_resolved = True
-                    purl = purl._replace(version=specifier.version)
+            resolved_purl = is_purl_resolved(purl = purl, specifiers= specifiers)
             dependent_packages.append(
                         models.DependentPackage(
-                        purl=str(purl),
+                        purl=str(resolved_purl.purl),
                         scope=scope,
                         is_runtime=True,
                         is_optional=False,
-                        is_resolved=is_resolved,
+                        is_resolved=resolved_purl.is_resolved,
                         extracted_requirement=req
                     )
                 )
         return dependent_packages
 
+
+def is_purl_resolved(purl: PackageURL, specifiers: SpecifierSet):
+    """
+    Check if the purl is resolved
+    """
+    is_resolved = False
+    if len(specifiers) == 1:
+        specifier = list(specifiers)[0]
+        if specifier.operator in ('==', '==='):
+            is_resolved = True
+            purl = purl._replace(version=specifier.version)
+    return ResolvedPurl(
+        purl=purl,
+        is_resolved=is_resolved,
+    )    
+
 class PipfileHandler(BaseDependencyFileHandler):
     datasource_id = 'pipfile'
     path_patterns = ('*Pipfile',)
@@ -1934,20 +1983,19 @@ def get_requirement_from_section(section, sub_section):
     """
     content = section.get(sub_section) or ""
     for req in content.splitlines():
-        if req:
-            #pytest-mypy >= 0.9.1; \
-            req = req.replace("; \\", "")
-            # pip>=19.1 # For proper file:// URLs support.
-            if "#" in req:
-                req , _ = req.rsplit("#")
+        if not req:
+            continue
+        #pytest-mypy >= 0.9.1; \
+        req = req.replace("; \\", "")
+        # pip>=19.1 # For proper file:// URLs support.
+        if "#" in req:
+            req , _ = req.rsplit("#")
+        try:
+            Requirement(req)
+            yield req
+        except:
             #pure-eval; black; tox
             req_split_by_semi_colon = req.split(";")
             req_split_by_semi_colon = [req.strip() for req in req_split_by_semi_colon if req]
-            if len(req_split_by_semi_colon) >= 2 and not(req_split_by_semi_colon[1].startswith("python_version") # pip>=19.1 ;python_version > 3.7 
-            or req_split_by_semi_colon[1].startswith("sys_platform") # pip>=19.1 ;sys_platform = "Windows"
-            or req_split_by_semi_colon[1].startswith("platform_system") # pip>=19.1 ;platform_system = "Windows"
-            or req_split_by_semi_colon[1].startswith("platform_python_implementation")):#pytest-black>=0.3.7; platform_python_implementation != "PyPy"
-                for temp_req in req_split_by_semi_colon:
-                    yield temp_req
-            else:
+            for req in req_split_by_semi_colon:
                 yield req

src/python_inspector/__init__.py

@@ -6,3 +6,5 @@
 # See https://github.com/nexB/scancode-toolkit for support or download.
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
+
+DEFAULT_PYTHON_VERSION = "3.8"

src/python_inspector/resolution.py

@@ -7,11 +7,11 @@
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
 
-import collections
 import operator
 import os
 import tarfile
 from typing import List
+from typing import NamedTuple
 from typing import Sequence
 from zipfile import ZipFile
 
@@ -32,7 +32,15 @@
 from _packagedcode.pypi import SetupCfgHandler
 from python_inspector import utils_pypi
 
-Candidate = collections.namedtuple("Candidate", "name version extras")
+
+class Candidate(NamedTuple):
+    """
+    A candidate is a package that can be installed.
+    """
+
+    name: str
+    version: str
+    extras: str
 
 
 def get_response(url):
@@ -68,9 +76,16 @@ def get_python_version_from_env_tag(python_version: str):
     return python_version
 
 
-def get_sdist_file(repos, candidate, python_version):
+def fetch_and_extract_sdist(repos, candidate, python_version):
     """
-    Return the sdist file for a candidate.
+    Fetch and extract the source distribution (sdist) for the ``candidate`` Candidate
+    from the `repos` list of PyPiRepository
+    and a required ``python_version`` Python version.
+    Return the directory location string where the sdist has been extracted.
+    Return None if the sdist was not fetched either
+    because does not exist in any of the ``repos`` or it does not work with
+    the required ``python_version``.
+    Raise an Exception if extraction fails.
     """
     sdist = utils_pypi.download_sdist(
         name=candidate.name,
@@ -82,25 +97,32 @@ def get_sdist_file(repos, candidate, python_version):
     if not sdist:
         return
 
-    sdist_file = None
-
     if sdist.endswith(".tar.gz"):
         sdist_file = sdist.rstrip(".tar.gz")
         with tarfile.open(os.path.join(utils_pypi.CACHE_THIRDPARTY_DIR, sdist)) as file:
             file.extractall(
                 os.path.join(utils_pypi.CACHE_THIRDPARTY_DIR, "extracted_sdists", sdist_file)
             )
-    if sdist.endswith(".zip"):
+    elif sdist.endswith(".zip"):
         sdist_file = sdist.rstrip(".zip")
         with ZipFile(os.path.join(utils_pypi.CACHE_THIRDPARTY_DIR, sdist)) as zip:
             zip.extractall(
                 os.path.join(utils_pypi.CACHE_THIRDPARTY_DIR, "extracted_sdists", sdist_file)
             )
 
-    if not sdist_file:
+    else:
         raise Exception(f"Unable to extract sdist {sdist}")
 
-    return sdist_file
+    return os.path.join(utils_pypi.CACHE_THIRDPARTY_DIR, "extracted_sdists", sdist_file, sdist_file)
+
+
+def remove_extras(identifier):
+    """
+    Return the identifier without extras.
+    >>> assert remove_extras("foo[bar]") == "foo"
+    """
+    name, _, _ = identifier.partition("[")
+    return name
 
 
 class PythonInputProvider(AbstractProvider):
@@ -212,32 +234,21 @@ def get_requirements_for_package_from_pypi_simple(self, candidate):
                 if dep.scope == "install":
                     yield packaging.requirements.Requirement(str(dep.extracted_requirement))
 
-        sdist_file = get_sdist_file(
+        sdist_file = fetch_and_extract_sdist(
             repos=self.repos, candidate=candidate, python_version=python_version
         )
 
         if sdist_file:
             setup_py_path = os.path.join(
-                utils_pypi.CACHE_THIRDPARTY_DIR,
-                "extracted_sdists",
-                sdist_file,
                 sdist_file,
                 "setup.py",
             )
             setup_cfg_path = os.path.join(
-                utils_pypi.CACHE_THIRDPARTY_DIR,
-                "extracted_sdists",
-                sdist_file,
                 sdist_file,
                 "setup.cfg",
             )
-            pkg_info_path = os.path.join(
-                utils_pypi.CACHE_THIRDPARTY_DIR, "extracted_sdists", sdist_file
-            )
+            pkg_info_path = os.path.dirname(sdist_file)
             requirement_path = os.path.join(
-                utils_pypi.CACHE_THIRDPARTY_DIR,
-                "extracted_sdists",
-                sdist_file,
                 sdist_file,
                 "requirements.txt",
             )
@@ -254,30 +265,31 @@ def get_requirements_for_package_from_pypi_simple(self, candidate):
                     continue
 
                 deps = list(handler.parse(path))
-                assert len(deps) == 1, handler
-                if not deps:
-                    continue
+                assert len(deps) == 1
+
                 dependencies = deps[0].dependencies
                 for dep in dependencies:
                     if not dep.purl:
                         continue
 
+                    if dep.scope != "install":
+                        continue
+
                     dep_purl = PackageURL.from_string(dep.purl)
-                    if not (
-                        dep.scope == "install"
-                        and (
-                            not (dep.is_resolved)
-                            or (dep.is_resolved and dep_purl.name not in self.resolved_requirements)
-                        )
-                    ):
+
+                    if self.is_dep_resolved_and_in_resolved_requirements(dep, dep_purl):
                         continue
+
                     if dep.is_resolved:
-                        self.resolved_requirements = (*self.resolved_requirements, dep_purl)
+                        self.resolved_requirements.append(dep_purl)
                     # skip the requirement starting with -- like
                     # --editable, --requirement
                     if not dep.extracted_requirement.startswith("--"):
                         yield packaging.requirements.Requirement(str(dep.extracted_requirement))
 
+    def is_dep_resolved_and_in_resolved_requirements(self, dep, dep_purl):
+        return dep.is_resolved and dep_purl.name in self.resolved_requirements
+
     def get_requirements_for_package_from_pypi_json_api(self, purl):
         """
         Return requirements for a package from the PyPI.org JSON API
@@ -308,7 +320,7 @@ def _iter_matches(self, identifier, requirements, incompatibilities):
         """
         Yield candidates for the given identifier, requirements and incompatibilities
         """
-        name, _, _ = identifier.partition("[")
+        name = remove_extras(identifier)
         bad_versions = {c.version for c in incompatibilities[identifier]}
         extras = {e for r in requirements[identifier] for e in r.extras}
         if not self.repos:
@@ -460,7 +472,7 @@ def get_resolved_dependencies(
     ]
     resolver = Resolver(
         provider=PythonInputProvider(
-            environment=environment, repos=repos, resolved_requirements=tuple(resolved_requirements)
+            environment=environment, repos=repos, resolved_requirements=resolved_requirements
         ),
         reporter=BaseReporter(),
     )

src/python_inspector/resolve_cli.py

@@ -168,7 +168,7 @@ def resolve_dependencies(
     direct_dependencies = []
 
     if PYPI_SIMPLE_URL not in index_urls:
-        index_urls = (*index_urls, PYPI_SIMPLE_URL)
+        index_urls = tuple([PYPI_SIMPLE_URL]) + tuple(index_urls)
 
     for req_file in requirement_files:
         deps = dependencies.get_dependencies_from_requirements(requirements_file=req_file)