Add tests and address review comments

Signed-off-by: Tushar Goel <tushar.goel.dav@gmail.com>

Author: TG1999

src/_packagedcode/pypi.py

@@ -741,6 +741,9 @@ def parse(cls, location):
 
     @classmethod
     def parse_reqs(cls, reqs, scope):
+        """
+        Parse a list of requirements and return a list of dependencies
+        """ 
         dependent_packages = []
         for req in reqs:
             is_resolved = False
@@ -1303,10 +1306,10 @@ def parse_with_dparse2(location, file_name=None):
     dependent_packages = []
 
     for dependency in dep_file.dependencies:
-        name = dependency.name
+        requirement = dependency.name
         is_resolved = False
         purl = PackageURL(type='pypi', name=dependency.name)
-        extracted_requirement = name
+
         # note: dparse2.dependencies.Dependency.specs comes from
         # packaging.requirements.Requirement.specifier
         # which in turn is a packaging.specifiers.SpecifierSet objects
@@ -1329,8 +1332,6 @@ def parse_with_dparse2(location, file_name=None):
                     is_resolved = True
                     purl = purl._replace(version=specifier.version)
 
-            extracted_requirement =  f"{name}{requirement}"
-
         dependent_packages.append(
             models.DependentPackage(
                 purl=purl.to_string(),
@@ -1339,7 +1340,7 @@ def parse_with_dparse2(location, file_name=None):
                 is_runtime=True,
                 is_optional=False,
                 is_resolved=is_resolved,
-                extracted_requirement=extracted_requirement
+                extracted_requirement=requirement
             )
         )
 
@@ -1936,22 +1937,22 @@ def get_requirement_from_section(section, sub_section):
     """
     Generate requirements from the `sub_section`
     """
-    content = section.get(sub_section)
-    if content:
-        for req in content.splitlines():
-            if req:
-                #pytest-mypy >= 0.9.1; \
-                req = req.replace("; \\", "")
-                # pip>=19.1 # For proper file:// URLs support.
-                if "#" in req:
-                    req , _ = req.rsplit("#")
-                #pure-eval; black; tox;
-                req_split_by_semi_colon = req.split(";")
-                req_split_by_semi_colon = [req.strip() for req in req_split_by_semi_colon]
-                if len(req_split_by_semi_colon) >= 2 and not(req_split_by_semi_colon[1].startswith("python_version") # pip>=19.1 ;python_version > 3.7 
-                or req_split_by_semi_colon[1].startswith("sys_platform") # pip>=19.1 ;sys_platform = "Windows"
-                or req_split_by_semi_colon[1].startswith("platform_system")): # pip>=19.1 ;platform_system = "Windows"
-                    for temp_req in req_split_by_semi_colon:
-                        yield temp_req
-                else:
-                    yield req
+    content = section.get(sub_section) or ""
+    for req in content.splitlines():
+        if req:
+            #pytest-mypy >= 0.9.1; \
+            req = req.replace("; \\", "")
+            # pip>=19.1 # For proper file:// URLs support.
+            if "#" in req:
+                req , _ = req.rsplit("#")
+            #pure-eval; black; tox
+            req_split_by_semi_colon = req.split(";")
+            req_split_by_semi_colon = [req.strip() for req in req_split_by_semi_colon if req]
+            if len(req_split_by_semi_colon) >= 2 and not(req_split_by_semi_colon[1].startswith("python_version") # pip>=19.1 ;python_version > 3.7 
+            or req_split_by_semi_colon[1].startswith("sys_platform") # pip>=19.1 ;sys_platform = "Windows"
+            or req_split_by_semi_colon[1].startswith("platform_system") # pip>=19.1 ;platform_system = "Windows"
+            or req_split_by_semi_colon[1].startswith("platform_python_implementation")):#pytest-black>=0.3.7; platform_python_implementation != "PyPy"
+                for temp_req in req_split_by_semi_colon:
+                    yield temp_req
+            else:
+                yield req

src/python_inspector/resolution.py

@@ -69,6 +69,35 @@ def get_python_version_from_env_tag(python_version: str):
     return python_version
 
 
+def get_sdist_file(repos, candidate):
+    """
+    Return the sdist file for a candidate.
+    """
+    sdist = utils_pypi.download_sdist(
+        name=candidate.name,
+        version=str(candidate.version),
+        repos=repos,
+    )
+    sdist_file = None
+
+    if sdist.endswith(".tar.gz"):
+        sdist_file = sdist.rstrip(".tar.gz")
+        with tarfile.open(os.path.join(utils_pypi.CACHE_THIRDPARTY_DIR, sdist)) as file:
+            file.extractall(
+                os.path.join(utils_pypi.CACHE_THIRDPARTY_DIR, "extracted_sdists", sdist_file)
+            )
+    if sdist.endswith(".zip"):
+        sdist_file = sdist.rstrip(".zip")
+        with ZipFile(os.path.join(utils_pypi.CACHE_THIRDPARTY_DIR, sdist)) as zip:
+            zip.extractall(
+                os.path.join(utils_pypi.CACHE_THIRDPARTY_DIR, "extracted_sdists", sdist_file)
+            )
+
+    if not sdist_file:
+        raise Exception(f"Unable to extract sdist {sdist}")
+    return sdist_file
+
+
 class PythonInputProvider(AbstractProvider):
     def __init__(self, environment=None, repos=tuple(), resolved_requirements=[]):
         self.environment = environment
@@ -170,7 +199,9 @@ def get_requirements_for_package(self, purl, candidate):
             return self.get_requirements_for_package_from_pypi_json_api(purl)
 
     def get_requirements_for_package_from_pypi_simple(self, candidate):
-
+        """
+        Return requirements for a package from the simple repositories.
+        """
         purl = PackageURL(type="pypi", name=candidate.name, version=str(candidate.version))
 
         formats = self.wheel_or_sdist_by_package[str(purl)]
@@ -194,28 +225,7 @@ def get_requirements_for_package_from_pypi_simple(self, candidate):
                             yield packaging.requirements.Requirement(str(dep.extracted_requirement))
 
             if format == "Sdist":
-                sdist = utils_pypi.download_sdist(
-                    name=candidate.name,
-                    version=str(candidate.version),
-                    repos=self.repos,
-                )
-                if sdist.endswith(".tar.gz"):
-                    sdist_file = sdist.rstrip(".tar.gz")
-                    file = tarfile.open(os.path.join(utils_pypi.CACHE_THIRDPARTY_DIR, sdist))
-                    file.extractall(
-                        os.path.join(
-                            utils_pypi.CACHE_THIRDPARTY_DIR, "extracted_sdists", sdist_file
-                        )
-                    )
-                    file.close()
-                if sdist.endswith(".zip"):
-                    sdist_file = sdist.rstrip(".zip")
-                    with ZipFile(os.path.join(utils_pypi.CACHE_THIRDPARTY_DIR, sdist), "r") as zip:
-                        zip.extractall(
-                            os.path.join(
-                                utils_pypi.CACHE_THIRDPARTY_DIR, "extracted_sdists", sdist_file
-                            )
-                        )
+                sdist_file = get_sdist_file(repos=self.repos, candidate=candidate)
                 setup_py_path = os.path.join(
                     utils_pypi.CACHE_THIRDPARTY_DIR,
                     "extracted_sdists",
@@ -241,48 +251,47 @@ def get_requirements_for_package_from_pypi_simple(self, candidate):
                     "requirements.txt",
                 )
 
-                path_by_format = {
-                    "pkginfo": pkg_info_path,
-                    "setup-py": setup_py_path,
-                    "setup-cfg": setup_cfg_path,
-                    "requirement": requirement_path,
+                path_by_sdist_parser = {
+                    PythonSdistPkgInfoFile: pkg_info_path,
+                    PythonSetupPyHandler: setup_py_path,
+                    SetupCfgHandler: setup_cfg_path,
+                    PipRequirementsFileHandler: requirement_path,
                 }
 
-                handler_by_format = {
-                    "pkginfo": PythonSdistPkgInfoFile,
-                    "setup-py": PythonSetupPyHandler,
-                    "setup-cfg": SetupCfgHandler,
-                    "requirement": PipRequirementsFileHandler,
-                }
-                for format in ["pkginfo", "setup-py", "setup-cfg", "requirement"]:
-                    path = path_by_format[format]
-                    if os.path.exists(path):
-                        handler = handler_by_format[format]
-                        deps = list(handler.parse(path))
-                        assert len(deps) == 1
-                        dependencies = deps[0].dependencies
-                        for dep in dependencies:
-                            # skip if no purl can be extracted for dependency
-                            if dep.purl:
-                                dep_purl = PackageURL.from_string(dep.purl)
-                                if dep.scope == "install" and (
-                                    not (dep.is_resolved)
-                                    or (
-                                        dep.is_resolved
-                                        and dep_purl.name not in self.resolved_requirements
-                                    )
-                                ):
-                                    if dep.is_resolved:
-                                        self.resolved_requirements.append(dep_purl)
-                                    # skip the requirement starting with -- like
-                                    # --editable, --requirement
-                                    if not dep.extracted_requirement.startswith("--"):
-                                        yield packaging.requirements.Requirement(
-                                            str(dep.extracted_requirement)
-                                        )
+                for handler, path in path_by_sdist_parser.items():
+                    if not os.path.exists(path):
+                        continue
 
-    def get_requirements_for_package_from_pypi_json_api(self, purl):
+                    deps = list(handler.parse(path))
+                    assert len(deps) == 1
+                    dependencies = deps[0].dependencies
+                    for dep in dependencies:
+                        if not dep.purl:
+                            continue
+
+                        dep_purl = PackageURL.from_string(dep.purl)
+                        if not (
+                            dep.scope == "install"
+                            and (
+                                not (dep.is_resolved)
+                                or (
+                                    dep.is_resolved
+                                    and dep_purl.name not in self.resolved_requirements
+                                )
+                            )
+                        ):
+                            continue
+                        if dep.is_resolved:
+                            self.resolved_requirements.append(dep_purl)
+                        # skip the requirement starting with -- like
+                        # --editable, --requirement
+                        if not dep.extracted_requirement.startswith("--"):
+                            yield packaging.requirements.Requirement(str(dep.extracted_requirement))
 
+    def get_requirements_for_package_from_pypi_json_api(self, purl):
+        """
+        Return requirements for a package from the PyPI.org JSON API
+        """
         # if no repos are provided use the incorrect but fast JSON API
         if str(purl) not in self.dependencies_by_purl:
             api_url = f"https://pypi.org/pypi/{purl.name}/{purl.version}/json"
@@ -455,11 +464,10 @@ def get_resolved_dependencies(
     If empty, use instead the PyPI.org JSON API exclusively instead
     """
     resolved_requirements = [
-        packaging.utils.canonicalize_name(r["requirement"].name)
+        packaging.utils.canonicalize_name(r.name)
         for r in requirements
-        if r["is_requirement_resolved"]
+        if getattr(r, "is_requirement_resolved", False)
     ]
-    requirements = [r["requirement"] for r in requirements]
     resolver = Resolver(
         provider=PythonInputProvider(
             environment=environment, repos=repos, resolved_requirements=resolved_requirements

src/python_inspector/resolve_cli.py

@@ -167,6 +167,9 @@ def resolve_dependencies(
     # TODO: deduplicate me
     direct_dependencies = []
 
+    if PYPI_SIMPLE_URL not in index_urls:
+        index_urls = (*index_urls, PYPI_SIMPLE_URL)
+
     for req_file in requirement_files:
         deps = dependencies.get_dependencies_from_requirements(requirements_file=req_file)
         for extra_data in dependencies.get_extra_data_from_requirements(requirements_file=req_file):
@@ -273,13 +276,14 @@ def resolve(direct_dependencies, environment, repos=tuple(), as_tree=False, max_
     Used the provided ``repos`` list of PypiSimpleRepository.
     If empty, use instead the PyPI.org JSON API exclusively.
     """
-    requirements = [
-        dict(
-            requirement=Requirement(requirement_string=d.extracted_requirement),
-            is_requirement_resolved=d.is_resolved,
-        )
-        for d in direct_dependencies
-    ]
+
+    requirements = []
+
+    for dependency in direct_dependencies:
+        requirement = Requirement(requirement_string=dependency.extracted_requirement)
+        requirement.is_requirement_resolved = dependency.is_resolved
+        requirements.append(requirement)
+
     resolved_dependencies = get_resolved_dependencies(
         requirements=requirements,
         environment=environment,