diff --git a/src/_packagedcode/models.py b/src/_packagedcode/models.py index ed3cf238..4fa67c00 100644 --- a/src/_packagedcode/models.py +++ b/src/_packagedcode/models.py @@ -373,6 +373,11 @@ class DependentPackage(ModelMixin): 'either from the datafile or collected from another source. Some ' 'lockfiles for Composer or Cargo contain extra dependency data.' ) + + extra_data = Mapping( + label='extra data', + help='A mapping of arbitrary extra data.', + ) @attr.attributes(slots=True) diff --git a/src/_packagedcode/pypi.py b/src/_packagedcode/pypi.py index c80247ed..a81b3817 100644 --- a/src/_packagedcode/pypi.py +++ b/src/_packagedcode/pypi.py @@ -10,6 +10,7 @@ import ast from configparser import ConfigParser +import copy import json import logging from pathlib import Path @@ -921,7 +922,7 @@ def get_requirements_txt_dependencies(location, include_nested=False): if req.name: # will be None if not pinned version = req.get_pinned_version - purl = PackageURL(type='pypi', name=req.name, version=version) + purl = PackageURL(type='pypi', name=canonicalize_name(req.name), version=version) else: # this is odd, but this can be null @@ -954,12 +955,41 @@ def get_requirements_txt_dependencies(location, include_nested=False): is_optional=is_optional, is_resolved=req.is_pinned or False, extracted_requirement=requirement, + extra_data=dict( + is_editable=req.is_editable, + link=req.link and req.link.url or None, + hash_options=req.hash_options or [], + is_constraint=req.is_constraint, + is_archive=req.is_archive, + is_wheel=req.is_wheel, + is_url=req.is_url, + is_vcs_url=req.is_vcs_url, + is_name_at_url=req.is_name_at_url, + is_local_path=req.is_local_path, + ), ) ) return dependent_packages, extra_data +def can_process_dependent_package(dep: models.DependentPackage): + """ + Return True if we can process the dependent package + typically anything that's not a plain standard specifier + can not be processed such as an editable requirement + """ + # copying dep.extra_data to avoid mutating the original + requirement_flags = copy.copy(dep.extra_data or {}) + requirement_flags.pop("hash_options", None) + if not requirement_flags: + return True + # we can not process the requirement if it has any flag set + # because this means it is not a standard specifier + # but rather some pip specific option of sorts + return not any(requirement_flags.values()) + + def get_attribute(metainfo, name, multiple=False): """ Return the value for the attribute ``name`` in the ``metainfo`` mapping, diff --git a/src/python_inspector/resolution.py b/src/python_inspector/resolution.py index dbdce29e..e37d18b1 100644 --- a/src/python_inspector/resolution.py +++ b/src/python_inspector/resolution.py @@ -10,27 +10,36 @@ import operator import os import tarfile +from typing import Dict +from typing import Generator from typing import List from typing import NamedTuple from typing import Sequence +from typing import Union from zipfile import ZipFile -import packaging.requirements import packaging.utils -import packaging.version import requests from packageurl import PackageURL from packaging.requirements import Requirement +from packaging.version import LegacyVersion +from packaging.version import Version +from packaging.version import parse as parse_version from resolvelib import AbstractProvider from resolvelib import Resolver from resolvelib.reporters import BaseReporter +from resolvelib.structs import DirectedGraph +from _packagedcode.models import DependentPackage +from _packagedcode.pypi import BasePypiHandler from _packagedcode.pypi import PipRequirementsFileHandler from _packagedcode.pypi import PypiWheelHandler -from _packagedcode.pypi import PythonSdistPkgInfoFile from _packagedcode.pypi import PythonSetupPyHandler from _packagedcode.pypi import SetupCfgHandler +from _packagedcode.pypi import can_process_dependent_package from python_inspector import utils_pypi +from python_inspector.utils_pypi import Environment +from python_inspector.utils_pypi import PypiSimpleRepository class Candidate(NamedTuple): @@ -43,17 +52,79 @@ class Candidate(NamedTuple): extras: str -def get_response(url): +class Result(NamedTuple): """ - Return a response for the given url. + Represent a dependency resolution result from resolvelib. + Below is the docstring copied from + https://github.com/sarugaku/resolvelib/blob/0e6ed4efa9ca079512ec999b54f2e5175a3b2111/src/resolvelib/resolvers.py#L454 + + The return value is a representation to the final resolution result. It + is a tuple subclass with three public members: + * ``mapping``: A dict of resolved candidates. Each key is an identifier + of a requirement (as returned by the provider's `identify` method), + and the value is the resolved candidate. + * ``graph``: A `DirectedGraph` instance representing the dependency tree. + The vertices are keys of `mapping`, and each edge represents *why* + a particular package is included. A special vertex `None` is + included to represent parents of user-supplied requirements. + * ``criteria``: A dict of "criteria" that hold detailed information on + how edges in the graph are derived. Each key is an identifier of a + requirement, and the value is a `Criterion` instance. + """ + + mapping: Dict + graph: DirectedGraph + criteria: Dict + + +def get_response(url: str) -> Dict: + """ + Return a mapping of the JSON response from fetching ``url`` + or None if the ``url`` cannot be fetched.. """ resp = requests.get(url) if resp.status_code == 200: return resp.json() - return None -def is_valid_version(parsed_version, requirements, identifier, bad_versions): +def get_requirements_from_distribution( + handler: BasePypiHandler, location: str +) -> List[Requirement]: + """ + Return a list of requirements from a source distribution or wheel at + ``location`` using the provided ``handler`` DatafileHandler for parsing. + """ + if not os.path.exists(location): + return [] + deps = list(handler.parse(location)) + assert len(deps) == 1 + return list(get_requirements_from_dependencies(dependencies=deps[0].dependencies)) + + +def is_requirements_file_in_setup_files(setup_files: List[str]) -> bool: + """ + Return True if the string ``requirements.txt`` is found in any of the ``setup_files`` location + strings to either a setup.py or setup.cfg file. + This is an indication that a requirements.txt is likely loaded in the setup.py + code and we use this as a hint to treat requirements.txt requirements + as being for the setup.py file. + """ + for setup_file in setup_files: + if not os.path.exists(setup_file): + continue + with open(setup_file, encoding="utf-8") as f: + # TODO also consider other file names + if "requirements.txt" in f.read(): + return True + return False + + +def is_valid_version( + parsed_version: Union[LegacyVersion, Version], + requirements: Dict, + identifier: str, + bad_versions: List[Version], +) -> bool: """ Return True if the parsed_version is valid for the given identifier. """ @@ -65,7 +136,7 @@ def is_valid_version(parsed_version, requirements, identifier, bad_versions): return True -def get_python_version_from_env_tag(python_version: str): +def get_python_version_from_env_tag(python_version: str) -> str: """ >>> assert get_python_version_from_env_tag("310") == "3.10" >>> assert get_python_version_from_env_tag("39") == "3.9" @@ -76,7 +147,9 @@ def get_python_version_from_env_tag(python_version: str): return python_version -def fetch_and_extract_sdist(repos, candidate, python_version): +def fetch_and_extract_sdist( + repos: List[PypiSimpleRepository], candidate: Candidate, python_version: str +) -> Union[str, None]: """ Fetch and extract the source distribution (sdist) for the ``candidate`` Candidate from the `repos` list of PyPiRepository @@ -116,7 +189,26 @@ def fetch_and_extract_sdist(repos, candidate, python_version): return os.path.join(utils_pypi.CACHE_THIRDPARTY_DIR, "extracted_sdists", sdist_file, sdist_file) -def remove_extras(identifier): +def get_requirements_from_dependencies(dependencies: List[DependentPackage]) -> List[Requirement]: + """ + Generate parsed requirements for the given ``dependencies``. + """ + for dep in dependencies: + if not dep.purl: + continue + + # TODO: consider other scopes and using the is_runtime flag + if dep.scope != "install": + continue + + # FIXME We are skipping editable requirements + # and other pip options for now + # https://github.com/nexB/python-inspector/issues/41 + if can_process_dependent_package(dep): + yield Requirement(str(dep.extracted_requirement)) + + +def remove_extras(identifier: str) -> str: """ Return the identifier without extras. >>> assert remove_extras("foo[bar]") == "foo" @@ -126,15 +218,14 @@ def remove_extras(identifier): class PythonInputProvider(AbstractProvider): - def __init__(self, environment=None, repos=tuple(), resolved_requirements=tuple()): + def __init__(self, environment=None, repos=tuple()): self.environment = environment self.repos = repos or [] self.versions_by_package = {} self.dependencies_by_purl = {} self.wheel_or_sdist_by_package = {} - self.resolved_requirements = resolved_requirements - def identify(self, requirement_or_candidate): + def identify(self, requirement_or_candidate: Union[Candidate, Requirement]) -> str: """Given a requirement, return an identifier for it. Overridden.""" name = packaging.utils.canonicalize_name(requirement_or_candidate.name) if requirement_or_candidate.extras: @@ -155,7 +246,9 @@ def get_preference( transitive = all(p is not None for _, p in information[identifier]) return transitive, identifier - def get_versions_for_package(self, name, repo=None): + def get_versions_for_package( + self, name: str, repo: Union[List[PypiSimpleRepository], None] = None + ) -> List[Version]: """ Return a list of versions for a package. """ @@ -164,14 +257,16 @@ def get_versions_for_package(self, name, repo=None): else: return self.get_versions_for_package_from_pypi_json_api(name) - def get_versions_for_package_from_repo(self, name, repo): + def get_versions_for_package_from_repo( + self, name: str, repo: PypiSimpleRepository + ) -> List[Version]: """ Return a list of versions for a package name from a repo """ versions = [] for version, package in repo.get_package_versions(name).items(): - python_version = packaging.version.parse( - get_python_version_from_env_tag(self.environment.python_version) + python_version = parse_version( + get_python_version_from_env_tag(python_version=self.environment.python_version) ) wheels = list(package.get_supported_wheels(environment=self.environment)) if wheels: @@ -186,7 +281,7 @@ def get_versions_for_package_from_repo(self, name, repo): versions.append(version) return versions - def get_versions_for_package_from_pypi_json_api(self, name): + def get_versions_for_package_from_pypi_json_api(self, name: str) -> List[Version]: """ Return a list of versions for a package name from the PyPI.org JSON API """ @@ -200,7 +295,9 @@ def get_versions_for_package_from_pypi_json_api(self, name): versions = self.versions_by_package[name] return versions - def get_requirements_for_package(self, purl, candidate): + def get_requirements_for_package( + self, purl: PackageURL, candidate: Candidate + ) -> Generator[Requirement, None, None]: """ Yield requirements for a package. """ @@ -209,12 +306,14 @@ def get_requirements_for_package(self, purl, candidate): else: return self.get_requirements_for_package_from_pypi_json_api(purl) - def get_requirements_for_package_from_pypi_simple(self, candidate): + def get_requirements_for_package_from_pypi_simple( + self, candidate: Candidate + ) -> List[Requirement]: """ Return requirements for a package from the simple repositories. """ - python_version = packaging.version.parse( - get_python_version_from_env_tag(self.environment.python_version) + python_version = parse_version( + get_python_version_from_env_tag(python_version=self.environment.python_version) ) wheels = utils_pypi.download_wheel( @@ -224,73 +323,67 @@ def get_requirements_for_package_from_pypi_simple(self, candidate): repos=self.repos, python_version=python_version, ) + + has_wheels = False + for wheel in wheels: - deps = list( - PypiWheelHandler.parse(os.path.join(utils_pypi.CACHE_THIRDPARTY_DIR, wheel)) + wheel_location = os.path.join(utils_pypi.CACHE_THIRDPARTY_DIR, wheel) + deps = get_requirements_from_distribution( + handler=PypiWheelHandler, + location=wheel_location, ) - assert len(deps) == 1 - deps = deps[0].dependencies - for dep in deps: - if dep.scope == "install": - yield packaging.requirements.Requirement(str(dep.extracted_requirement)) - - sdist_file = fetch_and_extract_sdist( - repos=self.repos, candidate=candidate, python_version=python_version - ) + if deps: + has_wheels = True + yield from deps - if sdist_file: - setup_py_path = os.path.join( - sdist_file, - "setup.py", - ) - setup_cfg_path = os.path.join( - sdist_file, - "setup.cfg", - ) - pkg_info_path = os.path.dirname(sdist_file) - requirement_path = os.path.join( - sdist_file, - "requirements.txt", + if not has_wheels: + sdist_location = fetch_and_extract_sdist( + repos=self.repos, candidate=candidate, python_version=python_version ) - path_by_sdist_parser = { - PythonSdistPkgInfoFile: pkg_info_path, - PythonSetupPyHandler: setup_py_path, - SetupCfgHandler: setup_cfg_path, - PipRequirementsFileHandler: requirement_path, - } - - for handler, path in path_by_sdist_parser.items(): - if not os.path.exists(path): - continue - - deps = list(handler.parse(path)) - assert len(deps) == 1 - - dependencies = deps[0].dependencies - for dep in dependencies: - if not dep.purl: - continue - - if dep.scope != "install": - continue - - dep_purl = PackageURL.from_string(dep.purl) - - if self.is_dep_resolved_and_in_resolved_requirements(dep, dep_purl): - continue - - if dep.is_resolved: - self.resolved_requirements.append(dep_purl) - # skip the requirement starting with -- like - # --editable, --requirement - if not dep.extracted_requirement.startswith("--"): - yield packaging.requirements.Requirement(str(dep.extracted_requirement)) - - def is_dep_resolved_and_in_resolved_requirements(self, dep, dep_purl): - return dep.is_resolved and dep_purl.name in self.resolved_requirements + if sdist_location: + setup_py_location = os.path.join( + sdist_location, + "setup.py", + ) + setup_cfg_location = os.path.join( + sdist_location, + "setup.cfg", + ) - def get_requirements_for_package_from_pypi_json_api(self, purl): + location_by_sdist_parser = { + PythonSetupPyHandler: setup_py_location, + SetupCfgHandler: setup_cfg_location, + } + + deps_in_setup = False + + for handler, location in location_by_sdist_parser.items(): + deps = get_requirements_from_distribution( + handler=handler, + location=location, + ) + if deps: + deps_in_setup = True + yield from deps + + requirement_location = os.path.join( + sdist_location, + "requirements.txt", + ) + if not deps_in_setup and is_requirements_file_in_setup_files( + setup_files=[setup_py_location, setup_cfg_location] + ): + deps = get_requirements_from_distribution( + handler=PipRequirementsFileHandler, + location=requirement_location, + ) + if deps: + yield from deps + + def get_requirements_for_package_from_pypi_json_api( + self, purl: PackageURL + ) -> List[Requirement]: """ Return requirements for a package from the PyPI.org JSON API """ @@ -304,38 +397,61 @@ def get_requirements_for_package_from_pypi_json_api(self, purl): requires_dist = info.get("requires_dist") or [] self.dependencies_by_purl[str(purl)] = requires_dist for dependency in self.dependencies_by_purl[str(purl)]: - yield packaging.requirements.Requirement(dependency) + yield Requirement(dependency) - def get_candidates(self, all_versions, requirements, identifier, bad_versions, name, extras): + def get_candidates( + self, + all_versions: List[str], + requirements: List[Requirement], + identifier: str, + bad_versions: List[str], + name: str, + extras: Dict, + ) -> Generator[Candidate, None, None]: """ Generate candidates for the given identifier. Overridden. """ for version in all_versions: - parsed_version = packaging.version.parse(version) - if not is_valid_version(parsed_version, requirements, identifier, bad_versions): + parsed_version = parse_version(version) + if not is_valid_version( + parsed_version=parsed_version, + requirements=requirements, + identifier=identifier, + bad_versions=bad_versions, + ): continue yield Candidate(name=name, version=parsed_version, extras=extras) - def _iter_matches(self, identifier, requirements, incompatibilities): + def _iter_matches( + self, + identifier: str, + requirements: List[Requirement], + incompatibilities: Dict, + ) -> Generator[Candidate, None, None]: """ Yield candidates for the given identifier, requirements and incompatibilities """ - name = remove_extras(identifier) + name = remove_extras(identifier=identifier) bad_versions = {c.version for c in incompatibilities[identifier]} extras = {e for r in requirements[identifier] for e in r.extras} if not self.repos: - all_versions = self.get_versions_for_package(name) + all_versions = self.get_versions_for_package(name=name) yield from self.get_candidates( all_versions, requirements, identifier, bad_versions, name, extras ) else: for repo in self.repos: - all_versions = self.get_versions_for_package(name, repo) + all_versions = self.get_versions_for_package(name=name, repo=repo) yield from self.get_candidates( all_versions, requirements, identifier, bad_versions, name, extras ) - def find_matches(self, identifier, requirements, incompatibilities): + def find_matches( + self, + identifier: str, + requirements: List[Requirement], + incompatibilities: Dict, + ) -> List[Candidate]: """Find all possible candidates that satisfy given constraints. Overridden.""" candidates = sorted( self._iter_matches(identifier, requirements, incompatibilities), @@ -344,11 +460,11 @@ def find_matches(self, identifier, requirements, incompatibilities): ) return candidates - def is_satisfied_by(self, requirement, candidate): + def is_satisfied_by(self, requirement: Requirement, candidate: Candidate) -> bool: """Whether the given requirement can be satisfied by a candidate. Overridden.""" return candidate.version in requirement.specifier - def _iter_dependencies(self, candidate): + def _iter_dependencies(self, candidate: Candidate) -> Generator[Requirement, None, None]: """ Yield dependencies for the given candidate. """ @@ -356,7 +472,7 @@ def _iter_dependencies(self, candidate): # TODO: handle extras https://github.com/nexB/python-inspector/issues/10 if candidate.extras: r = f"{name}=={candidate.version}" - yield packaging.requirements.Requirement(r) + yield Requirement(r) purl = PackageURL( type="pypi", @@ -364,7 +480,7 @@ def _iter_dependencies(self, candidate): version=str(candidate.version), ) - for r in self.get_requirements_for_package(purl, candidate): + for r in self.get_requirements_for_package(purl=purl, candidate=candidate): if r.marker is None: yield r else: @@ -372,19 +488,25 @@ def _iter_dependencies(self, candidate): { "extra": "", "python_version": get_python_version_from_env_tag( - self.environment.python_version + python_version=self.environment.python_version ), "platform_system": self.environment.operating_system.capitalize(), + "sys_platform": self.environment.operating_system, } ): yield r - def get_dependencies(self, candidate): + def get_dependencies(self, candidate: Candidate) -> List[Requirement]: """Get dependencies of a candidate. Overridden.""" return list(self._iter_dependencies(candidate)) -def get_wheel_download_urls(purl, repos, environment, python_version): +def get_wheel_download_urls( + purl: PackageURL, + repos: List[PypiSimpleRepository], + environment: Environment, + python_version: str, +) -> List[str]: """ Return a list of download urls for the given purl. """ @@ -399,7 +521,9 @@ def get_wheel_download_urls(purl, repos, environment, python_version): yield wheel.download_url -def get_sdist_download_url(purl, repos, python_version): +def get_sdist_download_url( + purl: PackageURL, repos: List[PypiSimpleRepository], python_version: str +) -> str: """ Return a list of download urls for the given purl. """ @@ -414,7 +538,7 @@ def get_sdist_download_url(purl, repos, python_version): return sdist.download_url -def get_all_srcs(mapping, graph): +def get_all_srcs(mapping: Dict, graph: DirectedGraph): """ Return a list of all sources in the graph. """ @@ -423,7 +547,7 @@ def get_all_srcs(mapping, graph): yield name -def dfs(mapping, graph, src): +def dfs(mapping: Dict, graph: DirectedGraph, src: str): """ Return a nested mapping of dependencies. """ @@ -442,7 +566,9 @@ def dfs(mapping, graph, src): ) -def format_resolution(results, environment, repos, as_tree=False): +def format_resolution( + results: Result, environment: Environment, repos: List[PypiSimpleRepository], as_tree=False +): """ Return a formatted resolution either as a tree or parent/children. """ @@ -503,8 +629,8 @@ def format_resolution(results, environment, repos, as_tree=False): def get_resolved_dependencies( requirements: List[Requirement], - environment: utils_pypi.Environment = None, - repos: Sequence[utils_pypi.PypiSimpleRepository] = tuple(), + environment: Environment = None, + repos: Sequence[PypiSimpleRepository] = tuple(), as_tree: bool = False, max_rounds: int = 200000, ): @@ -516,15 +642,8 @@ def get_resolved_dependencies( Used the provided ``repos`` list of PypiSimpleRepository. If empty, use instead the PyPI.org JSON API exclusively instead """ - resolved_requirements = [ - packaging.utils.canonicalize_name(r.name) - for r in requirements - if getattr(r, "is_requirement_resolved", False) - ] resolver = Resolver( - provider=PythonInputProvider( - environment=environment, repos=repos, resolved_requirements=resolved_requirements - ), + provider=PythonInputProvider(environment=environment, repos=repos), reporter=BaseReporter(), ) results = resolver.resolve(requirements=requirements, max_rounds=max_rounds) diff --git a/src/python_inspector/resolve_cli.py b/src/python_inspector/resolve_cli.py index 1c47595b..9babbe17 100644 --- a/src/python_inspector/resolve_cli.py +++ b/src/python_inspector/resolve_cli.py @@ -11,11 +11,14 @@ import json import sys +from typing import List import click from packaging.requirements import Requirement from tinynetrc import Netrc +from _packagedcode.models import DependentPackage +from _packagedcode.pypi import can_process_dependent_package from python_inspector import dependencies from python_inspector import utils from python_inspector import utils_pypi @@ -277,12 +280,7 @@ def resolve(direct_dependencies, environment, repos=tuple(), as_tree=False, max_ If empty, use instead the PyPI.org JSON API exclusively. """ - requirements = [] - - for dependency in direct_dependencies: - requirement = Requirement(requirement_string=dependency.extracted_requirement) - requirement.is_requirement_resolved = dependency.is_resolved - requirements.append(requirement) + requirements = list(get_requirements_from_direct_dependencies(direct_dependencies)) resolved_dependencies = get_resolved_dependencies( requirements=requirements, @@ -297,6 +295,18 @@ def resolve(direct_dependencies, environment, repos=tuple(), as_tree=False, max_ return initial_requirements, resolved_dependencies +def get_requirements_from_direct_dependencies( + direct_dependencies: List[DependentPackage], +) -> List[Requirement]: + for dependency in direct_dependencies: + # FIXME We are skipping editable requirements + # and other pip options for now + # https://github.com/nexB/python-inspector/issues/41 + if not can_process_dependent_package(dependency): + continue + yield Requirement(requirement_string=dependency.extracted_requirement) + + def write_output(headers, requirements, resolved_dependencies, json_output): """ Write headers, requirements and resolved_dependencies as JSON to ``json_output``. diff --git a/tests/data/default-url-expected.json b/tests/data/default-url-expected.json index 9aa07274..2dc22860 100644 --- a/tests/data/default-url-expected.json +++ b/tests/data/default-url-expected.json @@ -22,7 +22,8 @@ "is_runtime": true, "is_optional": false, "is_resolved": true, - "resolved_package": {} + "resolved_package": {}, + "extra_data": {} } ], "resolved_dependencies": [ diff --git a/tests/data/parse-reqs-with-setup_requires-and-python-requires.json b/tests/data/parse-reqs-with-setup_requires-and-python-requires.json index fe4e9cb4..27540efd 100644 --- a/tests/data/parse-reqs-with-setup_requires-and-python-requires.json +++ b/tests/data/parse-reqs-with-setup_requires-and-python-requires.json @@ -36,7 +36,8 @@ "is_runtime": true, "is_optional": false, "is_resolved": false, - "resolved_package": {} + "resolved_package": {}, + "extra_data": {} }, { "purl": "pkg:pypi/qtpy", @@ -45,7 +46,8 @@ "is_runtime": true, "is_optional": false, "is_resolved": false, - "resolved_package": {} + "resolved_package": {}, + "extra_data": {} }, { "purl": "pkg:pypi/setuptools", @@ -54,7 +56,8 @@ "is_runtime": true, "is_optional": false, "is_resolved": false, - "resolved_package": {} + "resolved_package": {}, + "extra_data": {} }, { "purl": "pkg:generic/python", @@ -63,7 +66,8 @@ "is_runtime": true, "is_optional": false, "is_resolved": false, - "resolved_package": {} + "resolved_package": {}, + "extra_data": {} }, { "purl": "pkg:pypi/pyside2", @@ -72,7 +76,8 @@ "is_runtime": true, "is_optional": false, "is_resolved": false, - "resolved_package": {} + "resolved_package": {}, + "extra_data": {} }, { "purl": "pkg:pypi/pyqt5", @@ -81,7 +86,8 @@ "is_runtime": true, "is_optional": false, "is_resolved": false, - "resolved_package": {} + "resolved_package": {}, + "extra_data": {} }, { "purl": "pkg:pypi/sphinx", @@ -90,7 +96,8 @@ "is_runtime": true, "is_optional": false, "is_resolved": false, - "resolved_package": {} + "resolved_package": {}, + "extra_data": {} }, { "purl": "pkg:pypi/restructuredtext-lint", @@ -99,7 +106,8 @@ "is_runtime": true, "is_optional": false, "is_resolved": false, - "resolved_package": {} + "resolved_package": {}, + "extra_data": {} }, { "purl": "pkg:pypi/pyside2", @@ -108,7 +116,8 @@ "is_runtime": true, "is_optional": false, "is_resolved": false, - "resolved_package": {} + "resolved_package": {}, + "extra_data": {} } ], "repository_homepage_url": null, diff --git a/tests/data/parse-reqs.json b/tests/data/parse-reqs.json index 64285c2c..67d6c278 100644 --- a/tests/data/parse-reqs.json +++ b/tests/data/parse-reqs.json @@ -36,7 +36,8 @@ "is_runtime": true, "is_optional": false, "is_resolved": false, - "resolved_package": {} + "resolved_package": {}, + "extra_data": {} }, { "purl": "pkg:pypi/attrs", @@ -45,7 +46,8 @@ "is_runtime": true, "is_optional": false, "is_resolved": false, - "resolved_package": {} + "resolved_package": {}, + "extra_data": {} }, { "purl": "pkg:pypi/beautifulsoup4", @@ -54,7 +56,8 @@ "is_runtime": true, "is_optional": false, "is_resolved": false, - "resolved_package": {} + "resolved_package": {}, + "extra_data": {} }, { "purl": "pkg:pypi/pip", @@ -63,7 +66,8 @@ "is_runtime": true, "is_optional": false, "is_resolved": false, - "resolved_package": {} + "resolved_package": {}, + "extra_data": {} }, { "purl": "pkg:pypi/click", @@ -72,7 +76,8 @@ "is_runtime": true, "is_optional": false, "is_resolved": false, - "resolved_package": {} + "resolved_package": {}, + "extra_data": {} }, { "purl": "pkg:pypi/intbitset", @@ -81,7 +86,8 @@ "is_runtime": true, "is_optional": false, "is_resolved": false, - "resolved_package": {} + "resolved_package": {}, + "extra_data": {} }, { "purl": "pkg:pypi/requests", @@ -90,7 +96,8 @@ "is_runtime": true, "is_optional": false, "is_resolved": false, - "resolved_package": {} + "resolved_package": {}, + "extra_data": {} }, { "purl": "pkg:pypi/saneyaml", @@ -99,7 +106,8 @@ "is_runtime": true, "is_optional": false, "is_resolved": false, - "resolved_package": {} + "resolved_package": {}, + "extra_data": {} }, { "purl": "pkg:pypi/text-unidecode", @@ -108,7 +116,8 @@ "is_runtime": true, "is_optional": false, "is_resolved": false, - "resolved_package": {} + "resolved_package": {}, + "extra_data": {} }, { "purl": "pkg:pypi/typing", @@ -117,7 +126,8 @@ "is_runtime": true, "is_optional": false, "is_resolved": false, - "resolved_package": {} + "resolved_package": {}, + "extra_data": {} }, { "purl": "pkg:pypi/pytest", @@ -126,7 +136,8 @@ "is_runtime": true, "is_optional": false, "is_resolved": false, - "resolved_package": {} + "resolved_package": {}, + "extra_data": {} }, { "purl": "pkg:pypi/pytest-xdist", @@ -135,7 +146,8 @@ "is_runtime": true, "is_optional": false, "is_resolved": false, - "resolved_package": {} + "resolved_package": {}, + "extra_data": {} }, { "purl": "pkg:pypi/aboutcode-toolkit", @@ -144,7 +156,8 @@ "is_runtime": true, "is_optional": false, "is_resolved": false, - "resolved_package": {} + "resolved_package": {}, + "extra_data": {} }, { "purl": "pkg:pypi/black", @@ -153,7 +166,8 @@ "is_runtime": true, "is_optional": false, "is_resolved": false, - "resolved_package": {} + "resolved_package": {}, + "extra_data": {} }, { "purl": "pkg:pypi/sphinx", @@ -162,7 +176,8 @@ "is_runtime": true, "is_optional": false, "is_resolved": false, - "resolved_package": {} + "resolved_package": {}, + "extra_data": {} }, { "purl": "pkg:pypi/sphinx-rtd-theme", @@ -171,7 +186,8 @@ "is_runtime": true, "is_optional": false, "is_resolved": false, - "resolved_package": {} + "resolved_package": {}, + "extra_data": {} }, { "purl": "pkg:pypi/doc8", @@ -180,7 +196,8 @@ "is_runtime": true, "is_optional": false, "is_resolved": false, - "resolved_package": {} + "resolved_package": {}, + "extra_data": {} } ], "repository_homepage_url": null, diff --git a/tests/data/pinned-requirements.txt b/tests/data/pinned-requirements.txt index cb292414..f3c5ff11 100644 --- a/tests/data/pinned-requirements.txt +++ b/tests/data/pinned-requirements.txt @@ -1,3 +1,4 @@ +aboutcode-toolkit==7.0.2 attrs==21.4.0 beautifulsoup4==4.11.1 certifi==2022.5.18.1 @@ -11,6 +12,7 @@ importlib-metadata==4.8.3 intbitset==3.0.1 packageurl-python==0.9.9 packaging==21.3 +-e git+https://github.com/nexB/python-inspector@18baae17824d6bacb4b1d519b10a0d0e50775884#egg=python_inspector pip-requirements-parser==31.2.0 pkginfo2==30.0.0 pyparsing==3.0.9 diff --git a/tests/data/pinned-requirements.txt-expected.json b/tests/data/pinned-requirements.txt-expected.json index 9e67935e..30a90c3e 100644 --- a/tests/data/pinned-requirements.txt-expected.json +++ b/tests/data/pinned-requirements.txt-expected.json @@ -14,6 +14,27 @@ "errors": [] }, "requirements": [ + { + "purl": "pkg:pypi/aboutcode-toolkit@7.0.2", + "extracted_requirement": "aboutcode-toolkit==7.0.2", + "scope": "install", + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "resolved_package": {}, + "extra_data": { + "is_editable": false, + "link": null, + "hash_options": [], + "is_constraint": false, + "is_archive": null, + "is_wheel": false, + "is_url": null, + "is_vcs_url": null, + "is_name_at_url": false, + "is_local_path": null + } + }, { "purl": "pkg:pypi/attrs@21.4.0", "extracted_requirement": "attrs==21.4.0", @@ -21,7 +42,19 @@ "is_runtime": true, "is_optional": false, "is_resolved": true, - "resolved_package": {} + "resolved_package": {}, + "extra_data": { + "is_editable": false, + "link": null, + "hash_options": [], + "is_constraint": false, + "is_archive": null, + "is_wheel": false, + "is_url": null, + "is_vcs_url": null, + "is_name_at_url": false, + "is_local_path": null + } }, { "purl": "pkg:pypi/beautifulsoup4@4.11.1", @@ -30,7 +63,19 @@ "is_runtime": true, "is_optional": false, "is_resolved": true, - "resolved_package": {} + "resolved_package": {}, + "extra_data": { + "is_editable": false, + "link": null, + "hash_options": [], + "is_constraint": false, + "is_archive": null, + "is_wheel": false, + "is_url": null, + "is_vcs_url": null, + "is_name_at_url": false, + "is_local_path": null + } }, { "purl": "pkg:pypi/certifi@2022.5.18.1", @@ -39,7 +84,19 @@ "is_runtime": true, "is_optional": false, "is_resolved": true, - "resolved_package": {} + "resolved_package": {}, + "extra_data": { + "is_editable": false, + "link": null, + "hash_options": [], + "is_constraint": false, + "is_archive": null, + "is_wheel": false, + "is_url": null, + "is_vcs_url": null, + "is_name_at_url": false, + "is_local_path": null + } }, { "purl": "pkg:pypi/charset-normalizer@2.0.12", @@ -48,7 +105,19 @@ "is_runtime": true, "is_optional": false, "is_resolved": true, - "resolved_package": {} + "resolved_package": {}, + "extra_data": { + "is_editable": false, + "link": null, + "hash_options": [], + "is_constraint": false, + "is_archive": null, + "is_wheel": false, + "is_url": null, + "is_vcs_url": null, + "is_name_at_url": false, + "is_local_path": null + } }, { "purl": "pkg:pypi/click@8.0.4", @@ -57,7 +126,19 @@ "is_runtime": true, "is_optional": false, "is_resolved": true, - "resolved_package": {} + "resolved_package": {}, + "extra_data": { + "is_editable": false, + "link": null, + "hash_options": [], + "is_constraint": false, + "is_archive": null, + "is_wheel": false, + "is_url": null, + "is_vcs_url": null, + "is_name_at_url": false, + "is_local_path": null + } }, { "purl": "pkg:pypi/colorama@0.4.4", @@ -66,7 +147,19 @@ "is_runtime": true, "is_optional": false, "is_resolved": true, - "resolved_package": {} + "resolved_package": {}, + "extra_data": { + "is_editable": false, + "link": null, + "hash_options": [], + "is_constraint": false, + "is_archive": null, + "is_wheel": false, + "is_url": null, + "is_vcs_url": null, + "is_name_at_url": false, + "is_local_path": null + } }, { "purl": "pkg:pypi/commoncode@30.2.0", @@ -75,7 +168,19 @@ "is_runtime": true, "is_optional": false, "is_resolved": true, - "resolved_package": {} + "resolved_package": {}, + "extra_data": { + "is_editable": false, + "link": null, + "hash_options": [], + "is_constraint": false, + "is_archive": null, + "is_wheel": false, + "is_url": null, + "is_vcs_url": null, + "is_name_at_url": false, + "is_local_path": null + } }, { "purl": "pkg:pypi/dparse2@0.6.1", @@ -84,7 +189,19 @@ "is_runtime": true, "is_optional": false, "is_resolved": true, - "resolved_package": {} + "resolved_package": {}, + "extra_data": { + "is_editable": false, + "link": null, + "hash_options": [], + "is_constraint": false, + "is_archive": null, + "is_wheel": false, + "is_url": null, + "is_vcs_url": null, + "is_name_at_url": false, + "is_local_path": null + } }, { "purl": "pkg:pypi/idna@3.3", @@ -93,7 +210,19 @@ "is_runtime": true, "is_optional": false, "is_resolved": true, - "resolved_package": {} + "resolved_package": {}, + "extra_data": { + "is_editable": false, + "link": null, + "hash_options": [], + "is_constraint": false, + "is_archive": null, + "is_wheel": false, + "is_url": null, + "is_vcs_url": null, + "is_name_at_url": false, + "is_local_path": null + } }, { "purl": "pkg:pypi/importlib-metadata@4.8.3", @@ -102,7 +231,19 @@ "is_runtime": true, "is_optional": false, "is_resolved": true, - "resolved_package": {} + "resolved_package": {}, + "extra_data": { + "is_editable": false, + "link": null, + "hash_options": [], + "is_constraint": false, + "is_archive": null, + "is_wheel": false, + "is_url": null, + "is_vcs_url": null, + "is_name_at_url": false, + "is_local_path": null + } }, { "purl": "pkg:pypi/intbitset@3.0.1", @@ -111,7 +252,19 @@ "is_runtime": true, "is_optional": false, "is_resolved": true, - "resolved_package": {} + "resolved_package": {}, + "extra_data": { + "is_editable": false, + "link": null, + "hash_options": [], + "is_constraint": false, + "is_archive": null, + "is_wheel": false, + "is_url": null, + "is_vcs_url": null, + "is_name_at_url": false, + "is_local_path": null + } }, { "purl": "pkg:pypi/packageurl-python@0.9.9", @@ -120,7 +273,19 @@ "is_runtime": true, "is_optional": false, "is_resolved": true, - "resolved_package": {} + "resolved_package": {}, + "extra_data": { + "is_editable": false, + "link": null, + "hash_options": [], + "is_constraint": false, + "is_archive": null, + "is_wheel": false, + "is_url": null, + "is_vcs_url": null, + "is_name_at_url": false, + "is_local_path": null + } }, { "purl": "pkg:pypi/packaging@21.3", @@ -129,7 +294,40 @@ "is_runtime": true, "is_optional": false, "is_resolved": true, - "resolved_package": {} + "resolved_package": {}, + "extra_data": { + "is_editable": false, + "link": null, + "hash_options": [], + "is_constraint": false, + "is_archive": null, + "is_wheel": false, + "is_url": null, + "is_vcs_url": null, + "is_name_at_url": false, + "is_local_path": null + } + }, + { + "purl": "pkg:pypi/python-inspector", + "extracted_requirement": "--editable git+https://github.com/nexB/python-inspector@18baae17824d6bacb4b1d519b10a0d0e50775884#egg=python_inspector", + "scope": "install", + "is_runtime": true, + "is_optional": false, + "is_resolved": false, + "resolved_package": {}, + "extra_data": { + "is_editable": true, + "link": "git+https://github.com/nexB/python-inspector@18baae17824d6bacb4b1d519b10a0d0e50775884#egg=python_inspector", + "hash_options": [], + "is_constraint": false, + "is_archive": false, + "is_wheel": false, + "is_url": true, + "is_vcs_url": true, + "is_name_at_url": false, + "is_local_path": true + } }, { "purl": "pkg:pypi/pip-requirements-parser@31.2.0", @@ -138,7 +336,19 @@ "is_runtime": true, "is_optional": false, "is_resolved": true, - "resolved_package": {} + "resolved_package": {}, + "extra_data": { + "is_editable": false, + "link": null, + "hash_options": [], + "is_constraint": false, + "is_archive": null, + "is_wheel": false, + "is_url": null, + "is_vcs_url": null, + "is_name_at_url": false, + "is_local_path": null + } }, { "purl": "pkg:pypi/pkginfo2@30.0.0", @@ -147,7 +357,19 @@ "is_runtime": true, "is_optional": false, "is_resolved": true, - "resolved_package": {} + "resolved_package": {}, + "extra_data": { + "is_editable": false, + "link": null, + "hash_options": [], + "is_constraint": false, + "is_archive": null, + "is_wheel": false, + "is_url": null, + "is_vcs_url": null, + "is_name_at_url": false, + "is_local_path": null + } }, { "purl": "pkg:pypi/pyparsing@3.0.9", @@ -156,7 +378,19 @@ "is_runtime": true, "is_optional": false, "is_resolved": true, - "resolved_package": {} + "resolved_package": {}, + "extra_data": { + "is_editable": false, + "link": null, + "hash_options": [], + "is_constraint": false, + "is_archive": null, + "is_wheel": false, + "is_url": null, + "is_vcs_url": null, + "is_name_at_url": false, + "is_local_path": null + } }, { "purl": "pkg:pypi/pyyaml@6.0", @@ -165,7 +399,19 @@ "is_runtime": true, "is_optional": false, "is_resolved": true, - "resolved_package": {} + "resolved_package": {}, + "extra_data": { + "is_editable": false, + "link": null, + "hash_options": [], + "is_constraint": false, + "is_archive": null, + "is_wheel": false, + "is_url": null, + "is_vcs_url": null, + "is_name_at_url": false, + "is_local_path": null + } }, { "purl": "pkg:pypi/requests@2.27.1", @@ -174,7 +420,19 @@ "is_runtime": true, "is_optional": false, "is_resolved": true, - "resolved_package": {} + "resolved_package": {}, + "extra_data": { + "is_editable": false, + "link": null, + "hash_options": [], + "is_constraint": false, + "is_archive": null, + "is_wheel": false, + "is_url": null, + "is_vcs_url": null, + "is_name_at_url": false, + "is_local_path": null + } }, { "purl": "pkg:pypi/resolvelib@0.8.1", @@ -183,7 +441,19 @@ "is_runtime": true, "is_optional": false, "is_resolved": true, - "resolved_package": {} + "resolved_package": {}, + "extra_data": { + "is_editable": false, + "link": null, + "hash_options": [], + "is_constraint": false, + "is_archive": null, + "is_wheel": false, + "is_url": null, + "is_vcs_url": null, + "is_name_at_url": false, + "is_local_path": null + } }, { "purl": "pkg:pypi/saneyaml@0.5.2", @@ -192,7 +462,19 @@ "is_runtime": true, "is_optional": false, "is_resolved": true, - "resolved_package": {} + "resolved_package": {}, + "extra_data": { + "is_editable": false, + "link": null, + "hash_options": [], + "is_constraint": false, + "is_archive": null, + "is_wheel": false, + "is_url": null, + "is_vcs_url": null, + "is_name_at_url": false, + "is_local_path": null + } }, { "purl": "pkg:pypi/soupsieve@2.3.2.post1", @@ -201,7 +483,19 @@ "is_runtime": true, "is_optional": false, "is_resolved": true, - "resolved_package": {} + "resolved_package": {}, + "extra_data": { + "is_editable": false, + "link": null, + "hash_options": [], + "is_constraint": false, + "is_archive": null, + "is_wheel": false, + "is_url": null, + "is_vcs_url": null, + "is_name_at_url": false, + "is_local_path": null + } }, { "purl": "pkg:pypi/text-unidecode@1.3", @@ -210,7 +504,19 @@ "is_runtime": true, "is_optional": false, "is_resolved": true, - "resolved_package": {} + "resolved_package": {}, + "extra_data": { + "is_editable": false, + "link": null, + "hash_options": [], + "is_constraint": false, + "is_archive": null, + "is_wheel": false, + "is_url": null, + "is_vcs_url": null, + "is_name_at_url": false, + "is_local_path": null + } }, { "purl": "pkg:pypi/toml@0.10.2", @@ -219,7 +525,19 @@ "is_runtime": true, "is_optional": false, "is_resolved": true, - "resolved_package": {} + "resolved_package": {}, + "extra_data": { + "is_editable": false, + "link": null, + "hash_options": [], + "is_constraint": false, + "is_archive": null, + "is_wheel": false, + "is_url": null, + "is_vcs_url": null, + "is_name_at_url": false, + "is_local_path": null + } }, { "purl": "pkg:pypi/typing@3.6.6", @@ -228,7 +546,19 @@ "is_runtime": true, "is_optional": false, "is_resolved": true, - "resolved_package": {} + "resolved_package": {}, + "extra_data": { + "is_editable": false, + "link": null, + "hash_options": [], + "is_constraint": false, + "is_archive": null, + "is_wheel": false, + "is_url": null, + "is_vcs_url": null, + "is_name_at_url": false, + "is_local_path": null + } }, { "purl": "pkg:pypi/typing-extensions@4.1.1", @@ -237,7 +567,19 @@ "is_runtime": true, "is_optional": false, "is_resolved": true, - "resolved_package": {} + "resolved_package": {}, + "extra_data": { + "is_editable": false, + "link": null, + "hash_options": [], + "is_constraint": false, + "is_archive": null, + "is_wheel": false, + "is_url": null, + "is_vcs_url": null, + "is_name_at_url": false, + "is_local_path": null + } }, { "purl": "pkg:pypi/urllib3@1.26.9", @@ -246,7 +588,19 @@ "is_runtime": true, "is_optional": false, "is_resolved": true, - "resolved_package": {} + "resolved_package": {}, + "extra_data": { + "is_editable": false, + "link": null, + "hash_options": [], + "is_constraint": false, + "is_archive": null, + "is_wheel": false, + "is_url": null, + "is_vcs_url": null, + "is_name_at_url": false, + "is_local_path": null + } }, { "purl": "pkg:pypi/zipp@3.6.0", @@ -255,10 +609,40 @@ "is_runtime": true, "is_optional": false, "is_resolved": true, - "resolved_package": {} + "resolved_package": {}, + "extra_data": { + "is_editable": false, + "link": null, + "hash_options": [], + "is_constraint": false, + "is_archive": null, + "is_wheel": false, + "is_url": null, + "is_vcs_url": null, + "is_name_at_url": false, + "is_local_path": null + } } ], "resolved_dependencies": [ + { + "package": "pkg:pypi/aboutcode-toolkit@7.0.2", + "dependencies": [ + "pkg:pypi/attrs@21.4.0", + "pkg:pypi/boolean-py@4.0", + "pkg:pypi/certifi@2022.5.18.1", + "pkg:pypi/click@8.0.4", + "pkg:pypi/jinja2@3.1.2", + "pkg:pypi/license-expression@30.0.0", + "pkg:pypi/openpyxl@3.0.10", + "pkg:pypi/packageurl-python@0.9.9", + "pkg:pypi/saneyaml@0.5.2" + ], + "wheel_urls": [ + "https://files.pythonhosted.org/packages/e8/94/7b1205e2992ea51e5caa6124e377772b1b538539a879b2d3838e47943087/aboutcode_toolkit-7.0.2-py3-none-any.whl" + ], + "sdist_url": "https://files.pythonhosted.org/packages/22/11/fb1a48393106b470fc05450a01317f22ca128541d5c5f755ce092ffaebd0/aboutcode-toolkit-7.0.2.tar.gz" + }, { "package": "pkg:pypi/attrs@21.4.0", "dependencies": [], @@ -278,20 +662,20 @@ "sdist_url": "https://files.pythonhosted.org/packages/e8/b0/cd2b968000577ec5ce6c741a54d846dfa402372369b8b6861720aa9ecea7/beautifulsoup4-4.11.1.tar.gz" }, { - "package": "pkg:pypi/certifi@2022.5.18.1", + "package": "pkg:pypi/boolean-py@4.0", "dependencies": [], "wheel_urls": [ - "https://files.pythonhosted.org/packages/11/dd/e015f3780f42dd9af62cf0107b44ea1298926627ecd70c17b0e484e95bcd/certifi-2022.5.18.1-py3-none-any.whl" + "https://files.pythonhosted.org/packages/3f/02/6389ef0529af6da0b913374dedb9bbde8eabfe45767ceec38cc37801b0bd/boolean.py-4.0-py3-none-any.whl" ], - "sdist_url": "https://files.pythonhosted.org/packages/07/10/75277f313d13a2b74fc56e29239d5c840c2bf09f17bf25c02b35558812c6/certifi-2022.5.18.1.tar.gz" + "sdist_url": "https://files.pythonhosted.org/packages/a2/d9/b6e56a303d221fc0bdff2c775e4eef7fedd58194aa5a96fa89fb71634cc9/boolean.py-4.0.tar.gz" }, { - "package": "pkg:pypi/chardet@4.0.0", + "package": "pkg:pypi/certifi@2022.5.18.1", "dependencies": [], "wheel_urls": [ - "https://files.pythonhosted.org/packages/19/c7/fa589626997dd07bd87d9269342ccb74b1720384a4d739a1872bd84fbe68/chardet-4.0.0-py2.py3-none-any.whl" + "https://files.pythonhosted.org/packages/11/dd/e015f3780f42dd9af62cf0107b44ea1298926627ecd70c17b0e484e95bcd/certifi-2022.5.18.1-py3-none-any.whl" ], - "sdist_url": "https://files.pythonhosted.org/packages/ee/2d/9cdc2b527e127b4c9db64b86647d567985940ac3698eeabc7ffaccb4ea61/chardet-4.0.0.tar.gz" + "sdist_url": "https://files.pythonhosted.org/packages/07/10/75277f313d13a2b74fc56e29239d5c840c2bf09f17bf25c02b35558812c6/certifi-2022.5.18.1.tar.gz" }, { "package": "pkg:pypi/charset-normalizer@2.0.12", @@ -322,12 +706,10 @@ "dependencies": [ "pkg:pypi/attrs@21.4.0", "pkg:pypi/beautifulsoup4@4.11.1", - "pkg:pypi/chardet@4.0.0", "pkg:pypi/click@8.0.4", "pkg:pypi/intbitset@3.0.1", "pkg:pypi/requests@2.27.1", "pkg:pypi/saneyaml@0.5.2", - "pkg:pypi/six@1.16.0", "pkg:pypi/text-unidecode@1.3" ], "wheel_urls": [ @@ -347,6 +729,14 @@ ], "sdist_url": "https://files.pythonhosted.org/packages/2d/a3/49cd39abdf8b4832cba27d635a775b721388e50671bee7ea0d10afcf0cfd/dparse2-0.6.1.tar.gz" }, + { + "package": "pkg:pypi/et-xmlfile@1.1.0", + "dependencies": [], + "wheel_urls": [ + "https://files.pythonhosted.org/packages/96/c2/3dd434b0108730014f1b96fd286040dc3bcb70066346f7e01ec2ac95865f/et_xmlfile-1.1.0-py3-none-any.whl" + ], + "sdist_url": "https://files.pythonhosted.org/packages/3d/5d/0413a31d184a20c763ad741cc7852a659bf15094c24840c5bdd1754765cd/et_xmlfile-1.1.0.tar.gz" + }, { "package": "pkg:pypi/idna@3.3", "dependencies": [], @@ -373,6 +763,44 @@ ], "sdist_url": "https://files.pythonhosted.org/packages/f8/80/81bf8129a2cc012a225ea0dc39cd6e11531a61d115616b1eead33b157164/intbitset-3.0.1.tar.gz" }, + { + "package": "pkg:pypi/jinja2@3.1.2", + "dependencies": [ + "pkg:pypi/markupsafe@2.1.1" + ], + "wheel_urls": [ + "https://files.pythonhosted.org/packages/bc/c3/f068337a370801f372f2f8f6bad74a5c140f6fda3d9de154052708dd3c65/Jinja2-3.1.2-py3-none-any.whl" + ], + "sdist_url": "https://files.pythonhosted.org/packages/7a/ff/75c28576a1d900e87eb6335b063fab47a8ef3c8b4d88524c4bf78f670cce/Jinja2-3.1.2.tar.gz" + }, + { + "package": "pkg:pypi/license-expression@30.0.0", + "dependencies": [ + "pkg:pypi/boolean-py@4.0" + ], + "wheel_urls": [ + "https://files.pythonhosted.org/packages/12/36/d973dc41ce58eb3e6c8833e75d9e50f8c8f6c9bc9ac7e4f5430294e88708/license_expression-30.0.0-py3-none-any.whl" + ], + "sdist_url": "https://files.pythonhosted.org/packages/80/ad/9b3614c4630f3a1c0d5d7e5f6cfa8e90850186864e1181e86a65150d983f/license-expression-30.0.0.tar.gz" + }, + { + "package": "pkg:pypi/markupsafe@2.1.1", + "dependencies": [], + "wheel_urls": [ + "https://files.pythonhosted.org/packages/fd/f4/524d2e8f5a3727cf309c2b7df7c732038375322df1376c9e9ef3aa92fcaf/MarkupSafe-2.1.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" + ], + "sdist_url": "https://files.pythonhosted.org/packages/1d/97/2288fe498044284f39ab8950703e88abbac2abbdf65524d576157af70556/MarkupSafe-2.1.1.tar.gz" + }, + { + "package": "pkg:pypi/openpyxl@3.0.10", + "dependencies": [ + "pkg:pypi/et-xmlfile@1.1.0" + ], + "wheel_urls": [ + "https://files.pythonhosted.org/packages/7b/60/9afac4fd6feee0ac09339de4101ee452ea643d26e9ce44c7708a0023f503/openpyxl-3.0.10-py2.py3-none-any.whl" + ], + "sdist_url": "https://files.pythonhosted.org/packages/2c/b8/ff77a718173fd73e49f883b4fda88f11af1fc51edb9252af3785b0cad987/openpyxl-3.0.10.tar.gz" + }, { "package": "pkg:pypi/packageurl-python@0.9.9", "dependencies": [], @@ -456,14 +884,6 @@ ], "sdist_url": "https://files.pythonhosted.org/packages/d1/4a/7374cac103bdfec7c199606784119bee9171ba051290b219ffec1bedc650/saneyaml-0.5.2.tar.gz" }, - { - "package": "pkg:pypi/six@1.16.0", - "dependencies": [], - "wheel_urls": [ - "https://files.pythonhosted.org/packages/d9/5a/e7c31adbe875f2abbb91bd84cf2dc52d792b5a01506781dbcf25c91daf11/six-1.16.0-py2.py3-none-any.whl" - ], - "sdist_url": "https://files.pythonhosted.org/packages/71/39/171f1c67cd00715f190ba0b100d606d440a28c93c7714febeca8b79af85e/six-1.16.0.tar.gz" - }, { "package": "pkg:pypi/soupsieve@2.3.2.post1", "dependencies": [], diff --git a/tests/data/single-url-except-simple-expected.json b/tests/data/single-url-except-simple-expected.json index b03f6a63..deb70dac 100644 --- a/tests/data/single-url-except-simple-expected.json +++ b/tests/data/single-url-except-simple-expected.json @@ -23,7 +23,8 @@ "is_runtime": true, "is_optional": false, "is_resolved": false, - "resolved_package": {} + "resolved_package": {}, + "extra_data": {} } ], "resolved_dependencies": [ @@ -42,7 +43,7 @@ "pkg:pypi/importlib-metadata@4.12.0", "pkg:pypi/itsdangerous@2.1.2", "pkg:pypi/jinja2@3.1.2", - "pkg:pypi/werkzeug@2.1.2" + "pkg:pypi/werkzeug@2.2.1" ], "wheel_urls": [ "https://files.pythonhosted.org/packages/af/6a/00d144ac1626fbb44c4ff36519712e258128985a5d0ae43344778ae5cbb9/Flask-2.1.3-py3-none-any.whl" @@ -86,12 +87,14 @@ "sdist_url": "https://files.pythonhosted.org/packages/1d/97/2288fe498044284f39ab8950703e88abbac2abbdf65524d576157af70556/MarkupSafe-2.1.1.tar.gz" }, { - "package": "pkg:pypi/werkzeug@2.1.2", - "dependencies": [], + "package": "pkg:pypi/werkzeug@2.2.1", + "dependencies": [ + "pkg:pypi/markupsafe@2.1.1" + ], "wheel_urls": [ - "https://files.pythonhosted.org/packages/c4/44/f50f2d22cdfb6d56c03d1b4cc3cfa03ebee2f21b59a7768f151e43415ba5/Werkzeug-2.1.2-py3-none-any.whl" + "https://files.pythonhosted.org/packages/c0/93/d6d60870e47162ea6a1bbdd787649eea776b2a70618dd66ed87cb2238543/Werkzeug-2.2.1-py3-none-any.whl" ], - "sdist_url": "https://files.pythonhosted.org/packages/10/cf/97eb1a3847c01ae53e8376bc21145555ac95279523a935963dc8ff96c50b/Werkzeug-2.1.2.tar.gz" + "sdist_url": "https://files.pythonhosted.org/packages/32/19/a92cdbd9fb795928dfca1031278ae8a7f051e78a2c057c224ad2d4cdd95e/Werkzeug-2.2.1.tar.gz" }, { "package": "pkg:pypi/zipp@3.8.1", diff --git a/tests/data/single-url-expected.json b/tests/data/single-url-expected.json index 34b600f5..1e5b9b76 100644 --- a/tests/data/single-url-expected.json +++ b/tests/data/single-url-expected.json @@ -22,7 +22,8 @@ "is_runtime": true, "is_optional": false, "is_resolved": true, - "resolved_package": {} + "resolved_package": {}, + "extra_data": {} } ], "resolved_dependencies": [ diff --git a/tests/data/tilde_req-expected.json b/tests/data/tilde_req-expected.json index 5af95dfd..5f6267f1 100644 --- a/tests/data/tilde_req-expected.json +++ b/tests/data/tilde_req-expected.json @@ -23,7 +23,8 @@ "is_runtime": true, "is_optional": false, "is_resolved": false, - "resolved_package": {} + "resolved_package": {}, + "extra_data": {} } ], "resolved_dependencies": [ diff --git a/tests/test_cli.py b/tests/test_cli.py index 4739dc0e..00ff1dc0 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -16,6 +16,8 @@ from click.testing import CliRunner from commoncode.testcase import FileDrivenTesting +from _packagedcode import models +from python_inspector.resolve_cli import get_requirements_from_direct_dependencies from python_inspector.resolve_cli import resolve_dependencies # Used for tests to regenerate fixtures with regen=True @@ -244,3 +246,65 @@ def run_cli(options, cli=resolve_dependencies, expected_rc=0, env=None): """ assert result.exit_code == expected_rc, error return result + + +def test_get_requirements_from_direct_dependencies(): + direct_dependencies = [ + models.DependentPackage( + purl="pkg:pypi/django", + scope="install", + is_runtime=True, + is_optional=False, + is_resolved=False, + extracted_requirement="django>=1.11.11", + extra_data=dict( + is_editable=False, + link=None, + hash_options=[], + is_constraint=False, + is_archive=False, + is_wheel=False, + is_url=False, + is_vcs_url=False, + is_name_at_url=False, + is_local_path=False, + ), + ) + ] + + requirements = [str(r) for r in get_requirements_from_direct_dependencies(direct_dependencies)] + + assert requirements == ["django>=1.11.11"] + + +def test_get_requirements_from_direct_dependencies_with_empty_list(): + assert list(get_requirements_from_direct_dependencies(direct_dependencies=[])) == [] + + +def test_get_requirements_from_direct_dependencies_with_editable_requirements(): + direct_dependencies = [ + models.DependentPackage( + purl="pkg:pypi/django", + scope="install", + is_runtime=True, + is_optional=False, + is_resolved=False, + extracted_requirement="django>=1.11.11", + extra_data=dict( + is_editable=True, + link=None, + hash_options=[], + is_constraint=False, + is_archive=False, + is_wheel=False, + is_url=False, + is_vcs_url=False, + is_name_at_url=False, + is_local_path=False, + ), + ) + ] + + requirements = [str(r) for r in get_requirements_from_direct_dependencies(direct_dependencies)] + + assert requirements == [] diff --git a/tests/test_pypi.py b/tests/test_pypi.py new file mode 100644 index 00000000..2b97e1ac --- /dev/null +++ b/tests/test_pypi.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# ScanCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/python-inspector for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +from _packagedcode import models +from _packagedcode.pypi import can_process_dependent_package + + +def test_can_process_dependent_package(): + dependency = models.DependentPackage( + purl="pkg:pypi/django", + scope="install", + is_runtime=True, + is_optional=False, + is_resolved=False, + extracted_requirement="django>=1.11.11", + extra_data=dict( + is_editable=False, + link=None, + hash_options=[], + is_constraint=False, + is_archive=False, + is_wheel=False, + is_url=False, + is_vcs_url=False, + is_name_at_url=False, + is_local_path=False, + ), + ) + + assert can_process_dependent_package(dependency) + + +def test_can_not_process_editable_dependent_package(): + dependency = models.DependentPackage( + purl="pkg:pypi/django", + scope="install", + is_runtime=True, + is_optional=False, + is_resolved=False, + extracted_requirement="django>=1.11.11", + extra_data=dict( + is_editable=True, + link=None, + hash_options=[], + is_constraint=False, + is_archive=False, + is_wheel=False, + is_url=False, + is_vcs_url=False, + is_name_at_url=False, + is_local_path=False, + ), + ) + + assert not can_process_dependent_package(dependency) + + +def test_can_process_dependent_package_without_extra_data(): + dependency = models.DependentPackage( + purl="pkg:pypi/django", + scope="install", + is_runtime=True, + is_optional=False, + is_resolved=False, + extracted_requirement="django>=1.11.11", + ) + + assert can_process_dependent_package(dependency) + + +def test_can_not_process_dependent_package_with_any_flags_set(): + dependency = models.DependentPackage( + purl="pkg:pypi/django", + scope="install", + is_runtime=True, + is_optional=False, + is_resolved=False, + extracted_requirement="django>=1.11.11", + extra_data=dict( + is_editable=True, + link="http://example.com/django.tar.gz", + hash_options=["--hash", "sha256:12345"], + is_constraint=True, + is_archive=True, + is_wheel=True, + is_url=True, + is_vcs_url=True, + is_name_at_url=True, + is_local_path=True, + ), + ) + + assert not can_process_dependent_package(dependency) diff --git a/tests/test_resolution.py b/tests/test_resolution.py index 97bb1b68..08c9d6fc 100644 --- a/tests/test_resolution.py +++ b/tests/test_resolution.py @@ -12,6 +12,8 @@ import pytest from packaging.requirements import Requirement +from _packagedcode import models +from python_inspector.resolution import get_requirements_from_dependencies from python_inspector.resolution import get_resolved_dependencies from python_inspector.resolution import is_valid_version from python_inspector.utils_pypi import PYPI_PUBLIC_REPO @@ -38,7 +40,7 @@ def test_get_resolved_dependencies_with_flask_and_python_310(): "pkg:pypi/itsdangerous@2.1.2", "pkg:pypi/jinja2@3.1.2", "pkg:pypi/markupsafe@2.1.1", - "pkg:pypi/werkzeug@2.1.2", + "pkg:pypi/werkzeug@2.2.1", ] @@ -63,7 +65,7 @@ def test_get_resolved_dependencies_with_flask_and_python_310_windows(): "pkg:pypi/itsdangerous@2.1.2", "pkg:pypi/jinja2@3.1.2", "pkg:pypi/markupsafe@2.1.1", - "pkg:pypi/werkzeug@2.1.2", + "pkg:pypi/werkzeug@2.2.1", ] @@ -116,7 +118,7 @@ def test_get_resolved_dependencies_with_tilde_requirement_using_json_api(): "pkg:pypi/itsdangerous@2.1.2", "pkg:pypi/jinja2@3.1.2", "pkg:pypi/markupsafe@2.1.1", - "pkg:pypi/werkzeug@2.1.2", + "pkg:pypi/werkzeug@2.2.1", "pkg:pypi/zipp@3.8.1", ] @@ -154,3 +156,65 @@ def test_is_valid_version(): bad_versions = [] identifier = "flask" assert is_valid_version(parsed_version, requirements, identifier, bad_versions) + + +def test_get_requirements_from_dependencies(): + dependencies = [ + models.DependentPackage( + purl="pkg:pypi/django", + scope="install", + is_runtime=True, + is_optional=False, + is_resolved=False, + extracted_requirement="django>=1.11.11", + extra_data=dict( + is_editable=False, + link=None, + hash_options=[], + is_constraint=False, + is_archive=False, + is_wheel=False, + is_url=False, + is_vcs_url=False, + is_name_at_url=False, + is_local_path=False, + ), + ) + ] + + requirements = [str(r) for r in get_requirements_from_dependencies(dependencies)] + + assert requirements == ["django>=1.11.11"] + + +def test_get_requirements_from_dependencies_with_empty_list(): + assert list(get_requirements_from_dependencies(dependencies=[])) == [] + + +def test_get_requirements_from_dependencies_with_editable_requirements(): + dependencies = [ + models.DependentPackage( + purl="pkg:pypi/django", + scope="install", + is_runtime=True, + is_optional=False, + is_resolved=False, + extracted_requirement="django>=1.11.11", + extra_data=dict( + is_editable=True, + link=None, + hash_options=[], + is_constraint=False, + is_archive=False, + is_wheel=False, + is_url=False, + is_vcs_url=False, + is_name_at_url=False, + is_local_path=False, + ), + ) + ] + + requirements = [str(r) for r in get_requirements_from_dependencies(dependencies)] + + assert requirements == []