Support both wheels and sdist

Signed-off-by: Tushar Goel <tushar.goel.dav@gmail.com>

Author: TG1999

src/_packagedcode/pypi.py

@@ -453,8 +453,9 @@ def parse_metadata(location, datasource_id, package_type):
         type=package_type,
         primary_language='Python',
         name=name,
-        version=version,
-        description=get_description(meta, location),
+        version=version, #TODO: https://github.com/nexB/scancode-toolkit/issues/3014
+        # description=get_description(meta, location),
+        description = "",
         declared_license=get_declared_license(meta),
         keywords=get_keywords(meta),
         parties=get_parties(meta),
@@ -1301,7 +1302,6 @@ def parse_with_dparse2(location, file_name=None):
     dependent_packages = []
 
     for dependency in dep_file.dependencies:
-        # print(dependency.serialize())
         name = dependency.name
         is_resolved = False
         purl = PackageURL(type='pypi', name=dependency.name)

src/python_inspector/resolution.py

@@ -27,6 +27,7 @@
 from resolvelib.reporters import BaseReporter
 
 from _packagedcode.pypi import PipRequirementsFileHandler
+from _packagedcode.pypi import PypiWheelHandler
 from _packagedcode.pypi import PythonSdistPkgInfoFile
 from _packagedcode.pypi import PythonSetupPyHandler
 from _packagedcode.pypi import SetupCfgHandler
@@ -74,6 +75,7 @@ def __init__(self, environment=None, repos=tuple()):
         self.repos = repos or []
         self.versions_by_package = {}
         self.dependencies_by_purl = {}
+        self.wheel_or_sdist_by_package = {}
 
     def identify(self, requirement_or_candidate):
         """Given a requirement, return an identifier for it. Overridden."""
@@ -109,17 +111,37 @@ def get_versions_for_package_from_repo(self, name, repo):
         """
         Return a list of versions for a package name from a repo
         """
+        versions = []
         for version, package in repo.get_package_versions(name).items():
+            name = packaging.utils.canonicalize_name(package.name)
+            purl = PackageURL(type="pypi", name=name, version=version)
+            python_version = packaging.version.parse(
+                get_python_version_from_env_tag(self.environment.python_version)
+            )
+            wheels = list(package.get_supported_wheels(environment=self.environment))
+            if wheels:
+                valid_wheel_present = False
+                for wheel in wheels:
+                    if wheel.requires_python and python_version in SpecifierSet(
+                        wheel.requires_python
+                    ):
+                        valid_wheel_present = True
+                    if not wheel.requires_python:
+                        valid_wheel_present = True
+                if valid_wheel_present:
+                    self.wheel_or_sdist_by_package[str(purl)] = "Wheel"
+                    versions.append(version)
+                    continue
             if package.sdist:
-                python_version = packaging.version.parse(
-                    get_python_version_from_env_tag(self.environment.python_version)
-                )
                 if package.sdist.requires_python and python_version in SpecifierSet(
                     package.sdist.requires_python
                 ):
-                    yield version
+                    self.wheel_or_sdist_by_package[str(purl)] = "Sdist"
+                    versions.append(version)
                 if not package.sdist.requires_python:
-                    yield version
+                    self.wheel_or_sdist_by_package[str(purl)] = "Sdist"
+                    versions.append(version)
+        return versions
 
     def get_versions_for_package_from_pypi_json_api(self, name):
         """
@@ -145,74 +167,98 @@ def get_requirements_for_package(self, purl, candidate):
             return self.get_requirements_for_package_from_pypi_json_api(purl)
 
     def get_requirements_for_package_from_pypi_simple(self, candidate):
-        sdist = utils_pypi.download_sdist(
-            name=candidate.name,
-            version=str(candidate.version),
-            repos=self.repos,
-        )
 
-        if sdist.endswith(".tar.gz"):
-            sdist_file = sdist.rstrip(".tar.gz")
-            file = tarfile.open(os.path.join(utils_pypi.CACHE_THIRDPARTY_DIR, sdist))
-            file.extractall(
-                os.path.join(utils_pypi.CACHE_THIRDPARTY_DIR, "extracted_sdists", sdist_file)
+        purl = PackageURL(type="pypi", name=candidate.name, version=str(candidate.version))
+
+        wheel_or_sdist = self.wheel_or_sdist_by_package[str(purl)]
+
+        if wheel_or_sdist == "Wheel":
+            wheels = utils_pypi.download_wheel(
+                name=candidate.name,
+                version=str(candidate.version),
+                environment=self.environment,
+                repos=self.repos,
             )
-            file.close()
-        if sdist.endswith(".zip"):
-            sdist_file = sdist.rstrip(".zip")
-            with ZipFile(os.path.join(utils_pypi.CACHE_THIRDPARTY_DIR, sdist), "r") as zip:
-                zip.extractall(
-                    os.path.join(utils_pypi.CACHE_THIRDPARTY_DIR, "extracted_sdists", sdist_file)
+            for wheel in wheels:
+                deps = list(
+                    PypiWheelHandler.parse(os.path.join(utils_pypi.CACHE_THIRDPARTY_DIR, wheel))
                 )
-        setup_py_path = os.path.join(
-            utils_pypi.CACHE_THIRDPARTY_DIR, "extracted_sdists", sdist_file, sdist_file, "setup.py"
-        )
-        setup_cfg_path = os.path.join(
-            utils_pypi.CACHE_THIRDPARTY_DIR, "extracted_sdists", sdist_file, sdist_file, "setup.cfg"
-        )
-        pkg_info_path = os.path.join(
-            utils_pypi.CACHE_THIRDPARTY_DIR, "extracted_sdists", sdist_file
-        )
-        requirement_path = os.path.join(
-            utils_pypi.CACHE_THIRDPARTY_DIR,
-            "extracted_sdists",
-            sdist_file,
-            sdist_file,
-            "requirements.txt",
-        )
-        pyproject_path = os.path.join(
-            utils_pypi.CACHE_THIRDPARTY_DIR,
-            "extracted_sdists",
-            sdist_file,
-            sdist_file,
-            "pyproject.toml",
-        )
-
-        path_by_format = {
-            "pkginfo": pkg_info_path,
-            "setup-py": setup_py_path,
-            "setup-cfg": setup_cfg_path,
-            "requirement": requirement_path,
-        }
-
-        handler_by_format = {
-            "pkginfo": PythonSdistPkgInfoFile,
-            "setup-py": PythonSetupPyHandler,
-            "setup-cfg": SetupCfgHandler,
-            "requirement": PipRequirementsFileHandler,
-        }
-        for format in ["pkginfo", "setup-py", "setup-cfg", "requirement"]:
-            path = path_by_format[format]
-            if os.path.exists(path):
-                handler = handler_by_format[format]
-                deps = list(handler.parse(path))
                 assert len(deps) == 1
-                dependencies = deps[0].dependencies
-                for dep in dependencies:
+                deps = deps[0].dependencies
+                for dep in deps:
                     if dep.scope == "install":
                         yield packaging.requirements.Requirement(str(dep.extracted_requirement))
-                if dependencies:
-                    break
+
+        if wheel_or_sdist == "Sdist":
+            sdist = utils_pypi.download_sdist(
+                name=candidate.name,
+                version=str(candidate.version),
+                repos=self.repos,
+            )
+
+            if sdist.endswith(".tar.gz"):
+                sdist_file = sdist.rstrip(".tar.gz")
+                file = tarfile.open(os.path.join(utils_pypi.CACHE_THIRDPARTY_DIR, sdist))
+                file.extractall(
+                    os.path.join(utils_pypi.CACHE_THIRDPARTY_DIR, "extracted_sdists", sdist_file)
+                )
+                file.close()
+            if sdist.endswith(".zip"):
+                sdist_file = sdist.rstrip(".zip")
+                with ZipFile(os.path.join(utils_pypi.CACHE_THIRDPARTY_DIR, sdist), "r") as zip:
+                    zip.extractall(
+                        os.path.join(
+                            utils_pypi.CACHE_THIRDPARTY_DIR, "extracted_sdists", sdist_file
+                        )
+                    )
+            setup_py_path = os.path.join(
+                utils_pypi.CACHE_THIRDPARTY_DIR,
+                "extracted_sdists",
+                sdist_file,
+                sdist_file,
+                "setup.py",
+            )
+            setup_cfg_path = os.path.join(
+                utils_pypi.CACHE_THIRDPARTY_DIR,
+                "extracted_sdists",
+                sdist_file,
+                sdist_file,
+                "setup.cfg",
+            )
+            pkg_info_path = os.path.join(
+                utils_pypi.CACHE_THIRDPARTY_DIR, "extracted_sdists", sdist_file
+            )
+            requirement_path = os.path.join(
+                utils_pypi.CACHE_THIRDPARTY_DIR,
+                "extracted_sdists",
+                sdist_file,
+                sdist_file,
+                "requirements.txt",
+            )
+
+            path_by_format = {
+                "pkginfo": pkg_info_path,
+                "setup-py": setup_py_path,
+                "setup-cfg": setup_cfg_path,
+                "requirement": requirement_path,
+            }
+
+            handler_by_format = {
+                "pkginfo": PythonSdistPkgInfoFile,
+                "setup-py": PythonSetupPyHandler,
+                "setup-cfg": SetupCfgHandler,
+                "requirement": PipRequirementsFileHandler,
+            }
+            for format in ["pkginfo", "setup-py", "setup-cfg", "requirement"]:
+                path = path_by_format[format]
+                if os.path.exists(path):
+                    handler = handler_by_format[format]
+                    deps = list(handler.parse(path))
+                    assert len(deps) == 1
+                    dependencies = deps[0].dependencies
+                    for dep in dependencies:
+                        if dep.scope == "install":
+                            yield packaging.requirements.Requirement(str(dep.extracted_requirement))
 
     def get_requirements_for_package_from_pypi_json_api(self, purl):
 
@@ -242,7 +288,7 @@ def _iter_matches(self, identifier, requirements, incompatibilities):
         """
         Yield candidates for the given identifier, requirements and incompatibilities
         """
-        name, _, _extras = identifier.partition("[")
+        name, _, _ = identifier.partition("[")
         bad_versions = {c.version for c in incompatibilities[identifier]}
         extras = {e for r in requirements[identifier] for e in r.extras}
         if not self.repos:

src/python_inspector/utils_pypi.py

@@ -1426,7 +1426,6 @@ def _get_package_versions_map(
                     verbose=verbose,
                     echo_func=echo_func,
                 )
-                # print(links)
                 # note that this is sorted so the mapping is also sorted
                 versions = {
                     package.version: package
@@ -1510,7 +1509,6 @@ def fetch_links(
             verbose=verbose,
             echo_func=echo_func,
         )
-        # print(text)
         soup = BeautifulSoup(text, features="html.parser")
         anchor_tags = soup.find_all("a")
         links_with_requires_python = []

tests/test_resolution.py

@@ -137,6 +137,7 @@ def test_without_supported_wheels():
         "pkg:pypi/cryptography@37.0.3",
         "pkg:pypi/hyperlink@21.0.0",
         "pkg:pypi/idna@3.3",
+        "pkg:pypi/pycparser@2.21",
         "pkg:pypi/setuptools@62.6.0",
         "pkg:pypi/txaio@22.2.1",
     ]