diff --git a/Makefile b/Makefile index 4a16890f..ddcabcec 100644 --- a/Makefile +++ b/Makefile @@ -18,7 +18,7 @@ dev: isort: @echo "-> Apply isort changes to ensure proper imports ordering" - ${VENV}/bin/isort --sl src tests + ${VENV}/bin/isort --sl -l 100 src tests black: @echo "-> Apply black code formatter" @@ -32,9 +32,10 @@ valid: isort black check: @echo "-> Run pycodestyle (PEP8) validation" - @${ACTIVATE} pycodestyle --max-line-length=100 --exclude=venv,lib,thirdparty,docs,migrations,settings.py . + @${ACTIVATE} pycodestyle --max-line-length=110 \ + --exclude=.eggs,etc/scripts,src/_packagedcode,venv,lib,thirdparty,docs . @echo "-> Run isort imports ordering validation" - @${ACTIVATE} isort --sl --check-only . + @${ACTIVATE} isort --sl --check-only -l 100 src tests @echo "-> Run black validation" @${ACTIVATE} black --check -l 100 diff --git a/README.rst b/README.rst index 8ea50de2..ce50be6a 100644 --- a/README.rst +++ b/README.rst @@ -24,6 +24,19 @@ The goal of python-inspector is to be a comprehensive library that can handle every style of Python package layouts, manifests and lockfiles. +Usage +-------- + +- Install with pip:: + + pip install python-inspector + +- Run a command line with:: + + dad --help + + + Its companion libraries are: - ``pip-requirements-parser``, a mostly correct pip requirements parsing diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 6ca19c4d..4a299116 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -11,15 +11,16 @@ jobs: parameters: job_name: ubuntu18_cpython image_name: ubuntu-18.04 - python_versions: ['3.6', '3.7', '3.8', '3.9', '3.10'] + python_versions: ['3.8', '3.9', '3.10'] test_suites: - all: venv/bin/pytest -n 2 -vvs + all: | + venv/bin/pytest -n 2 -vvs - template: etc/ci/azure-posix.yml parameters: job_name: ubuntu20_cpython image_name: ubuntu-20.04 - python_versions: ['3.6', '3.7', '3.8', '3.9', '3.10'] + python_versions: ['3.8', '3.9', '3.10'] test_suites: all: venv/bin/pytest -n 2 -vvs @@ -27,7 +28,7 @@ jobs: parameters: job_name: macos1015_cpython image_name: macos-10.15 - python_versions: ['3.6', '3.7', '3.8', '3.9', '3.10'] + python_versions: ['3.8', '3.9', '3.10'] test_suites: all: venv/bin/pytest -n 2 -vvs @@ -35,7 +36,7 @@ jobs: parameters: job_name: macos11_cpython image_name: macos-11 - python_versions: ['3.7', '3.8', '3.9', '3.10'] + python_versions: ['3.8', '3.9', '3.10'] test_suites: all: venv/bin/pytest -n 2 -vvs @@ -43,7 +44,7 @@ jobs: parameters: job_name: win2019_cpython image_name: windows-2019 - python_versions: ['3.6', '3.7', '3.8', '3.9', '3.10'] + python_versions: ['3.8', '3.9', '3.10'] test_suites: all: venv\Scripts\pytest -n 2 -vvs @@ -51,6 +52,6 @@ jobs: parameters: job_name: win2022_cpython image_name: windows-2022 - python_versions: ['3.7', '3.8', '3.9', '3.10'] + python_versions: ['3.8', '3.9', '3.10'] test_suites: all: venv\Scripts\pytest -n 2 -vvs diff --git a/docs/source/dependencies-design.rst b/docs/source/dependencies-design.rst index 7b0c9d63..952f3817 100644 --- a/docs/source/dependencies-design.rst +++ b/docs/source/dependencies-design.rst @@ -21,11 +21,11 @@ For each dependency, there can be a version "requirement" that can be an exact version, a version expression (aka. version specifier) as defined in `pep-0440 `__ with additional OS and environment tags and constraints as specified in -`pep-0508 `__ . +`pep-0508 `__ . -In particular the required Python version of a package (or version specifier) -can be set for the whole package (with the``python_requires`` attribute) or as -a marker for a given direct or indirect dependency. pip processes requirement +In particular the required Python version of a package (or version specifier) +can be set for the whole package (with the``python_requires`` attribute) or as +a marker for a given direct or indirect dependency. pip processes requirement specifiers and constraints from a "requirements" file and internally resolves dependency versions recursively by querying the PyPI Python package index repository at https://PyPI.org @@ -129,7 +129,7 @@ And a flat list of unique dependencies would be: - thing 3.0 - shebang 1.0 -The implementation may likely be similar to +The implementation may likely be similar to `pipgrip `__, but using the `resolvelib `__ library as used and vendored in pip instead of an implementation of @@ -139,7 +139,7 @@ The expected benefit of this tool is a simpler way to resolve Python dependencies that will not require complex installation of a Python toolchain specific to a given project environment when the goal is only to resolve dependencies. In particular the key new capability is to run this tool on a -single Python version and resolve versions for alternative Python versions, +single Python version and resolve versions for alternative Python versions, operating systems and architectures without having to install all the packages in the dependency tree. @@ -158,7 +158,8 @@ The outline of the processing is to: - For each top-level requirement (e.g. name/version): - Fetch all the corresponding versions metadata using the PyPI API(s) - - Fetch the packages as needed to further obtain the next-level dependencies, and this recursively + - Fetch the packages as needed to further obtain the next-level + dependencies, and this recursively - Resolve a correct dependency version for each name. - Dump JSON @@ -168,7 +169,7 @@ User experience: ---------------- The goal of the command line interface and user experience is to be -obvious and familiar to a pip user. +obvious and familiar to a pip user. Create a new CLI named "dad" short for "dad analyzes dependencies" with these key options: @@ -352,4 +353,3 @@ ScanCode Toolkit can detect the and normalize the declared licenses in package metadata and also collect and normalize all the metadata. This could be a refinement for later. - diff --git a/etc/scripts/utils_requirements.py b/etc/scripts/utils_requirements.py index 7c99a33b..db7e0ee2 100644 --- a/etc/scripts/utils_requirements.py +++ b/etc/scripts/utils_requirements.py @@ -8,6 +8,7 @@ # See https://github.com/nexB/skeleton for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # +import os import re import subprocess diff --git a/requirements-dev.txt b/requirements-dev.txt index e69de29b..6514e9e2 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -0,0 +1,41 @@ +aboutcode-toolkit==7.0.2 +black==22.3.0 +bleach==4.1.0 +boolean.py==4.0 +cffi==1.15.0 +cryptography==37.0.2 +dataclasses==0.8 +docutils==0.18.1 +et-xmlfile==1.1.0 +execnet==1.9.0 +importlib-resources==5.4.0 +iniconfig==1.1.1 +isort==5.10.1 +jeepney==0.7.1 +jinja2==3.0.3 +keyring==23.4.1 +license-expression==30.0.0 +markupsafe==2.0.1 +mypy-extensions==0.4.3 +openpyxl==3.0.10 +pathspec==0.9.0 +pkginfo==1.8.3 +platformdirs==2.4.0 +pluggy==1.0.0 +py==1.11.0 +pycodestyle==2.8.0 +pycparser==2.21 +pygments==2.12.0 +pytest==7.0.1 +pytest-forked==1.4.0 +pytest-xdist==2.5.0 +readme-renderer==34.0 +requests-toolbelt==0.9.1 +rfc3986==1.5.0 +secretstorage==3.3.2 +six==1.16.0 +tomli==1.2.3 +tqdm==4.64.0 +twine==3.8.0 +typed-ast==1.5.4 +webencodings==0.5.1 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index e69de29b..cb292414 100644 --- a/requirements.txt +++ b/requirements.txt @@ -0,0 +1,27 @@ +attrs==21.4.0 +beautifulsoup4==4.11.1 +certifi==2022.5.18.1 +charset-normalizer==2.0.12 +click==8.0.4 +colorama==0.4.4 +commoncode==30.2.0 +dparse2==0.6.1 +idna==3.3 +importlib-metadata==4.8.3 +intbitset==3.0.1 +packageurl-python==0.9.9 +packaging==21.3 +pip-requirements-parser==31.2.0 +pkginfo2==30.0.0 +pyparsing==3.0.9 +PyYAML==6.0 +requests==2.27.1 +resolvelib==0.8.1 +saneyaml==0.5.2 +soupsieve==2.3.2.post1 +text-unidecode==1.3 +toml==0.10.2 +typing==3.6.6 +typing_extensions==4.1.1 +urllib3==1.26.9 +zipp==3.6.0 diff --git a/src/_packagedcode/models.py b/src/_packagedcode/models.py index 11911657..ec4faf72 100644 --- a/src/_packagedcode/models.py +++ b/src/_packagedcode/models.py @@ -356,6 +356,12 @@ class DependentPackage(ModelMixin): "lockfiles for Composer or Cargo contain extra dependency data.", ) + # dependencies = List( + # item_type="DependentPackage", + # label="dependencies", + # help="A list of DependentPackage for this package.", + # ) + @attr.attributes(slots=True) class Dependency(DependentPackage): diff --git a/src/_packagedcode/pypi.py b/src/_packagedcode/pypi.py index 1f73d3c8..9cdef5b2 100644 --- a/src/_packagedcode/pypi.py +++ b/src/_packagedcode/pypi.py @@ -812,7 +812,7 @@ def get_requires_dependencies(requires, default_scope="install"): is_runtime=True, is_optional=False, is_resolved=is_resolved, - extracted_requirement=requirement, + extracted_requirement=str(req), ) ) diff --git a/src/python_inspector/resolution.py b/src/python_inspector/resolution.py new file mode 100644 index 00000000..14529dea --- /dev/null +++ b/src/python_inspector/resolution.py @@ -0,0 +1,331 @@ +# +# Copyright (c) Tzu-ping Chung , nexB Inc. and others. +# SPDX-License-Identifier: ISC AND Apache-2.0 +# derived and heavily modified from https://github.com/sarugaku/resolvelib + +# See https://github.com/nexB/python-inspector for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +import collections +import operator +import os +from typing import List +from typing import Sequence + +import packaging.requirements +import packaging.utils +import packaging.version +import requests +from packageurl import PackageURL +from packaging.requirements import Requirement +from resolvelib import AbstractProvider +from resolvelib import Resolver +from resolvelib.reporters import BaseReporter + +from _packagedcode.pypi import PypiWheelHandler +from python_inspector import utils_pypi + +Candidate = collections.namedtuple("Candidate", "name version extras") + + +def get_response(url): + """ + Return a response for the given url. + """ + resp = requests.get(url) + if resp.status_code == 200: + return resp.json() + return None + + +def is_valid_version(parsed_version, requirements, identifier, bad_versions): + """ + Return True if the parsed_version is valid for the given identifier. + """ + if ( + any(parsed_version not in r.specifier for r in requirements[identifier]) + or parsed_version in bad_versions + ): + return False + return True + + +def get_python_version_from_env_tag(python_version: str): + """ + >>> assert get_python_version_from_env_tag("310") == "3.10" + >>> assert get_python_version_from_env_tag("39") == "3.9" + """ + elements = list(python_version) + elements.insert(1, ".") + python_version = "".join(elements) + return python_version + + +class PythonInputProvider(AbstractProvider): + def __init__(self, environment=None, repos=tuple()): + self.environment = environment + self.repos = repos or [] + self.versions_by_package = {} + self.dependencies_by_purl = {} + + def identify(self, requirement_or_candidate): + """Given a requirement, return an identifier for it. Overridden.""" + name = packaging.utils.canonicalize_name(requirement_or_candidate.name) + if requirement_or_candidate.extras: + extras_str = ",".join(sorted(requirement_or_candidate.extras)) + return "{}[{}]".format(name, extras_str) + return name + + def get_preference( + self, + identifier, + resolutions, + candidates, + information, + backtrack_causes, + ): + """Produce a sort key for given requirement based on preference. Overridden.""" + + transitive = all(p is not None for _, p in information[identifier]) + return transitive, identifier + + def get_versions_for_package(self, name, repo=None): + """ + Return a list of versions for a package. + """ + if repo and self.environment: + return self.get_versions_for_package_from_repo(name, repo) + else: + return self.get_versions_for_package_from_pypi_json_api(name) + + def get_versions_for_package_from_repo(self, name, repo): + """ + Return a list of versions for a package name from a repo + """ + versions = [] + for version, package in repo._get_package_versions_map(name).items(): + wheels = package.get_supported_wheels(environment=self.environment) + if list(wheels): + versions.append(version) + return versions + + def get_versions_for_package_from_pypi_json_api(self, name): + """ + Return a list of versions for a package name from the PyPI.org JSON API + """ + if name not in self.versions_by_package: + api_url = f"https://pypi.org/pypi/{name}/json" + resp = get_response(api_url) + if not resp: + self.versions_by_package[name] = [] + releases = resp.get("releases") or {} + self.versions_by_package[name] = releases.keys() or [] + versions = self.versions_by_package[name] + return versions + + def get_requirements_for_package(self, purl, candidate): + """ + Yield requirements for a package. + """ + if self.repos and self.environment: + return self.get_requirements_for_package_from_pypi_simple(candidate) + else: + return self.get_requirements_for_package_from_pypi_json_api(purl) + + def get_requirements_for_package_from_pypi_simple(self, candidate): + wheels = utils_pypi.download_wheel( + name=candidate.name, + version=str(candidate.version), + environment=self.environment, + repos=self.repos, + ) + for wheel in wheels: + deps = list( + PypiWheelHandler.parse(os.path.join(utils_pypi.CACHE_THIRDPARTY_DIR, wheel)) + ) + assert len(deps) == 1 + deps = deps[0].dependencies + for dep in deps: + if dep.scope == "install": + yield packaging.requirements.Requirement(str(dep.extracted_requirement)) + + def get_requirements_for_package_from_pypi_json_api(self, purl): + + # if no repos are provided use the incorrect but fast JSON API + if str(purl) not in self.dependencies_by_purl: + api_url = f"https://pypi.org/pypi/{purl.name}/{purl.version}/json" + resp = get_response(api_url) + if not resp: + self.dependencies_by_purl[str(purl)] = [] + info = resp.get("info") or {} + requires_dist = info.get("requires_dist") or [] + self.dependencies_by_purl[str(purl)] = requires_dist + for dependency in self.dependencies_by_purl[str(purl)]: + yield packaging.requirements.Requirement(dependency) + + def get_candidates(self, all_versions, requirements, identifier, bad_versions, name, extras): + """ + Generate candidates for the given identifier. Overridden. + """ + for version in all_versions: + parsed_version = packaging.version.parse(version) + if not is_valid_version(parsed_version, requirements, identifier, bad_versions): + continue + yield Candidate(name=name, version=parsed_version, extras=extras) + + def _iter_matches(self, identifier, requirements, incompatibilities): + """ + Yield candidates for the given identifier, requirements and incompatibilities + """ + name, _, _extras = identifier.partition("[") + bad_versions = {c.version for c in incompatibilities[identifier]} + extras = {e for r in requirements[identifier] for e in r.extras} + if not self.repos: + all_versions = self.get_versions_for_package(name) + yield from self.get_candidates( + all_versions, requirements, identifier, bad_versions, name, extras + ) + else: + for repo in self.repos: + all_versions = self.get_versions_for_package(name, repo) + yield from self.get_candidates( + all_versions, requirements, identifier, bad_versions, name, extras + ) + + def find_matches(self, identifier, requirements, incompatibilities): + """Find all possible candidates that satisfy given constraints. Overridden.""" + candidates = sorted( + self._iter_matches(identifier, requirements, incompatibilities), + key=operator.attrgetter("version"), + reverse=True, + ) + return candidates + + def is_satisfied_by(self, requirement, candidate): + """Whether the given requirement can be satisfied by a candidate. Overridden.""" + return candidate.version in requirement.specifier + + def _iter_dependencies(self, candidate): + """ + Yield dependencies for the given candidate. + """ + name = packaging.utils.canonicalize_name(candidate.name) + # TODO: handle extras https://github.com/nexB/python-inspector/issues/10 + if candidate.extras: + r = f"{name}=={candidate.version}" + yield packaging.requirements.Requirement(r) + + purl = PackageURL( + type="pypi", + name=name, + version=str(candidate.version), + ) + + for r in self.get_requirements_for_package(purl, candidate): + if r.marker is None: + yield r + else: + if r.marker.evaluate( + { + "extra": "", + "python_version": get_python_version_from_env_tag( + self.environment.python_version + ), + "platform_system": self.environment.operating_system.capitalize(), + } + ): + yield r + + def get_dependencies(self, candidate): + """Get dependencies of a candidate. Overridden.""" + return list(self._iter_dependencies(candidate)) + + +def get_all_srcs(mapping, graph): + """ + Return a list of all sources in the graph. + """ + for name in mapping.keys(): + if list(graph.iter_parents(name)) == [None]: + yield name + + +def dfs(mapping, graph, src): + """ + Return a nested mapping of dependencies. + """ + children = list(graph.iter_children(src)) + src_purl = PackageURL( + type="pypi", + name=src, + version=str(mapping[src].version), + ) + if not children: + return dict(package=str(src_purl), dependencies=[]) + + return dict( + package=str(src_purl), + dependencies=sorted([dfs(mapping, graph, c) for c in children], key=lambda d: d["package"]), + ) + + +def format_resolution(results, as_tree=False): + """ + Return a formatted resolution either as a tree or parent/children. + """ + mapping = results.mapping + graph = results.graph + + if not as_tree: + as_parent_children = [] + parents = mapping.keys() + for parent in parents: + parent_purl = PackageURL( + type="pypi", + name=parent, + version=str(mapping[parent].version), + ) + dependencies = [] + for dependency in graph.iter_children(parent): + dep_purl = PackageURL( + type="pypi", + name=dependency, + version=str(mapping[dependency].version), + ) + dependencies.append(str(dep_purl)) + dependencies.sort() + parent_children = dict(package=str(parent_purl), dependencies=dependencies) + as_parent_children.append(parent_children) + as_parent_children.sort(key=lambda d: d["package"]) + return as_parent_children + else: + dependencies = [] + for src in get_all_srcs(mapping=mapping, graph=graph): + dependencies.append(dfs(mapping=mapping, graph=graph, src=src)) + + dependencies.sort(key=lambda d: d["package"]) + return dependencies + + +def get_resolved_dependencies( + requirements: List[Requirement], + environment: utils_pypi.Environment = None, + repos: Sequence[utils_pypi.PypiSimpleRepository] = tuple(), + as_tree: bool = False, +): + """ + Return resolved dependencies of a ``requirements`` list of Requirement for + an ``enviroment`` Environment. The resolved dependencies are formatted as + parent/children or a nested tree if ``as_tree`` is True. + + Used the provided ``repos`` list of PypiSimpleRepository. + If empty, use instead the PyPI.org JSON API exclusively instead + """ + resolver = Resolver( + provider=PythonInputProvider(environment=environment, repos=repos), + reporter=BaseReporter(), + ) + results = resolver.resolve(requirements=requirements) + results = format_resolution(results, as_tree=as_tree) + return results diff --git a/src/python_inspector/resolution.py.ABOUT b/src/python_inspector/resolution.py.ABOUT new file mode 100644 index 00000000..6c23ef8b --- /dev/null +++ b/src/python_inspector/resolution.py.ABOUT @@ -0,0 +1,14 @@ +about_resource: resolution.py +package_url: pkg:github.com/sarugaku/resolvelib/@a5ae68140afac49dd1a1a8e87eff9550db4a586b#tests/functional/python/test_resolvers_python.py +type: github +namespace: sarugaku +name: resolvelib +version: a5ae68140afac49dd1a1a8e87eff9550db4a586b +subpath: tests/functional/python/test_resolvers_python.py + +download_url: https://github.com/sarugaku/resolvelib/blob/a5ae68140afac49dd1a1a8e87eff9550db4a586b/tests/functional/python/test_resolvers_python.py +copyright: Copyright (c) 2018, Tzu-ping Chung +license_expression: isc +notes: The PythonInputProvider is copied in part and heavily modified from + resolvelib tests/functional/python/test_resolvers_python.py + \ No newline at end of file diff --git a/src/python_inspector/resolution.py.LICENSE b/src/python_inspector/resolution.py.LICENSE new file mode 100644 index 00000000..b9077766 --- /dev/null +++ b/src/python_inspector/resolution.py.LICENSE @@ -0,0 +1,13 @@ +Copyright (c) 2018, Tzu-ping Chung + +Permission to use, copy, modify, and distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. diff --git a/src/python_inspector/resolve_cli.py b/src/python_inspector/resolve_cli.py index f7363541..bf6dd7eb 100644 --- a/src/python_inspector/resolve_cli.py +++ b/src/python_inspector/resolve_cli.py @@ -13,13 +13,20 @@ import sys import click +from packaging.requirements import Requirement from python_inspector import dependencies from python_inspector import utils_pypi from python_inspector.cli_utils import FileOptionType +from python_inspector.resolution import get_resolved_dependencies TRACE = False +__version__ = "0.5.0" + +DEFAULT_PYTHON_VERSION = "38" +PYPI_SIMPLE_URL = "https://pypi.org/simple" + @click.command() @click.option( @@ -41,7 +48,7 @@ metavar="SPECIFIER", multiple=True, required=False, - help="Package specifier such as django==1.2.3. " "This option can be used multiple times.", + help="Package specifier such as django==1.2.3. This option can be used multiple times.", ) @click.option( "-p", @@ -49,8 +56,7 @@ "python_version", type=click.Choice(utils_pypi.PYTHON_VERSIONS), metavar="PYVER", - # TODO: Make default the current Python version - default="38", # utils_pypi.PYTHON_VERSIONS, + default=DEFAULT_PYTHON_VERSION, show_default=True, help="Python version to use for dependency resolution.", ) @@ -62,17 +68,17 @@ metavar="OS", default="linux", show_default=True, - help="OS to use for dependency resolution. ", + help="OS to use for dependency resolution.", ) @click.option( "--index-url", "index_urls", type=str, metavar="INDEX", - default=utils_pypi.PYPI_INDEX_URLS, show_default=True, + default=tuple([PYPI_SIMPLE_URL]), multiple=True, - help="PyPI index URL(s) to use in order of preference. " + help="PyPI simple index URL(s) to use in order of preference. " "This option can be used multiple times.", ) @click.option( @@ -87,7 +93,14 @@ @click.option( "--use-cached-index", is_flag=True, - help="Use cached on-disk PyPI package indexes and do not refetch if present.", + hidden=True, + help="Use cached on-disk PyPI simple package indexes and do not refetch if present.", +) +@click.option( + "--use-pypi-json-api", + is_flag=True, + help="Use PyPI JSON API to fetch dependency data. Faster but not always correct. " + "--index-url are ignored when this option is active.", ) @click.option( "--debug", @@ -103,7 +116,8 @@ def resolve_dependencies( operating_system, index_urls, json_output, - use_cached_index, + use_cached_index=False, + use_pypi_json_api=False, debug=TRACE, ): """ @@ -111,17 +125,21 @@ def resolve_dependencies( and SPECIFIER(s) and save the results as JSON to FILE. Resolve the dependencies for the requested ``--python-version`` PYVER and - ``--operating_system`` OS combination defaulting to the current version and OS. + ``--operating_system`` OS combination defaulting Python version 3.8 and + linux OS. + + Download from the provided PyPI simple --index-url INDEX(s) URLs defaulting + to PyPI.org - Download from the provided PyPI simple --index-url INDEX(s) URLs. Error and progress are printed to stderr. - For example:: - dad --spec "flask" --requirement etc/scripts/requirements.txt --json - + For example, display the results of resolving the dependencies for flask==2.1.2 + on screen:: + + dad --spec "flask==2.1.2" --json - """ - # FIXME: Use stderr and click.secho - print(f"Resolving dependencies...") + click.secho(f"Resolving dependencies...") # TODO: deduplicate me direct_dependencies = [] @@ -135,13 +153,13 @@ def resolve_dependencies( direct_dependencies.append(dep) if not direct_dependencies: - print("Error: no requirements requested.") + click.secho("Error: no requirements requested.") sys.exit(1) if debug: - print("direct_dependencies:") + click.secho("direct_dependencies:") for dep in direct_dependencies: - print(" ", dep) + click.secho(f" {dep}") # create a resolution environments environment = utils_pypi.Environment.from_pyver_and_os( @@ -149,58 +167,108 @@ def resolve_dependencies( ) if debug: - print("environment:", environment) + click.secho(f"environment: {environment}") - # Collect PyPI repos repos = [] - for index_url in index_urls: - index_url = index_url.strip("/") - existing = utils_pypi.DEFAULT_PYPI_REPOS_BY_URL.get(index_url) - if existing: - existing.use_cached_index = use_cached_index - repos.append(existing) - else: - repo = utils_pypi.PypiSimpleRepository( - index_url=index_url, - use_cached_index=use_cached_index, - ) - repos.append(repo) + if not use_pypi_json_api: + # Collect PyPI repos + for index_url in index_urls: + index_url = index_url.strip("/") + existing = utils_pypi.DEFAULT_PYPI_REPOS_BY_URL.get(index_url) + if existing: + existing.use_cached_index = use_cached_index + repos.append(existing) + else: + repo = utils_pypi.PypiSimpleRepository( + index_url=index_url, + use_cached_index=use_cached_index, + ) + repos.append(repo) if debug: - print("repos:") + click.secho("repos:") for repo in repos: - print(" ", repo) + click.secho(f" {repo}") # resolve dependencies proper - resolved_dependencies = resolve(direct_dependencies) - write_output(results=resolved_dependencies, json_output=json_output) + requirements, resolved_dependencies = resolve( + direct_dependencies=direct_dependencies, + environment=environment, + repos=repos, + as_tree=False, + ) + + cli_options = [f"--requirement {rf}" for rf in requirement_files] + cli_options += [f"--specifier {sp}" for sp in specifiers] + cli_options += [f"--index-url {iu}" for iu in index_urls] + cli_options += [f"--python-version {python_version}"] + cli_options += [f"--operating-system {operating_system}"] + cli_options += ["--json "] + + notice = ( + "Dependency tree generated with python-inspector.\n" + "python-inspector is a free software tool from nexB Inc. and others.\n" + "Visit https://github.com/nexB/scancode-toolkit/ for support and download." + ) + + headers = dict( + tool_name="dad", + tool_homepageurl="https://github.com/nexB/python-inspector", + tool_version=__version__, + options=cli_options, + notice=notice, + warnings=[], + errors=[], + ) + + write_output( + headers=headers, + requirements=requirements, + resolved_dependencies=resolved_dependencies, + json_output=json_output, + ) if debug: - print("done!") + click.secho("done!") -def resolve(direct_dependencies): +def resolve(direct_dependencies, environment, repos=tuple(), as_tree=False): """ Resolve dependencies given a ``direct_dependencies`` list of - DependentPackage and return SOMETHING TBD. + DependentPackage and return a tuple of (initial_requirements, + resolved_dependencies). + Used the provided ``repos`` list of PypiSimpleRepository. + If empty, use instead the PyPI.org JSON API exclusively. """ - from packaging.requirements import Requirement - reqs = [Requirement(d.extracted_requirement) for d in direct_dependencies] - - return dict( - headers=[dict(tool="dad")], - dependencies=[d.to_dict() for d in direct_dependencies], - requirements=[str(r) for r in reqs], + requirements = [ + Requirement(requirement_string=d.extracted_requirement) for d in direct_dependencies + ] + resolved_dependencies = get_resolved_dependencies( + requirements=requirements, + environment=environment, + repos=repos, + as_tree=as_tree, ) + initial_requirements = [d.to_dict() for d in direct_dependencies] + + return initial_requirements, resolved_dependencies + -def write_output(results, json_output): +def write_output(headers, requirements, resolved_dependencies, json_output): """ - Write headers, and resolved dependency results to ``output_file`` + Write headers, requirements and resolved_dependencies as JSON to ``json_output``. + Return the output data. """ - # TODO : create tree, add headers - json.dump(results, json_output, indent=2) + output = dict( + headers=headers, + requirements=requirements, + resolved_dependencies=resolved_dependencies, + ) + + json.dump(output, json_output, indent=2) + return output if __name__ == "__main__": diff --git a/src/python_inspector/utils_pip_compatibility_tags.py b/src/python_inspector/utils_pip_compatibility_tags.py index 59b9b8b4..122ae59a 100644 --- a/src/python_inspector/utils_pip_compatibility_tags.py +++ b/src/python_inspector/utils_pip_compatibility_tags.py @@ -49,13 +49,14 @@ def _mac_platforms(arch): if match: name, major, minor, actual_arch = match.groups() mac_version = (int(major), int(minor)) + len_mac = len("macosx_") arches = [ # Since we have always only checked that the platform starts # with "macosx", for backwards-compatibility we extract the # actual prefix provided by the user in case they provided # something like "macosxcustom_". It may be good to remove # this as undocumented or deprecate it in the future. - "{}_{}".format(name, arch[len("macosx_") :]) + "{}_{}".format(name, arch[len_mac:]) for arch in mac_platforms(mac_version, actual_arch) ] else: diff --git a/src/python_inspector/utils_pypi.py b/src/python_inspector/utils_pypi.py index ec5a9795..484d5cec 100644 --- a/src/python_inspector/utils_pypi.py +++ b/src/python_inspector/utils_pypi.py @@ -164,7 +164,6 @@ def get_python_dot_version(version): CACHE_THIRDPARTY_DIR = ".cache/thirdparty" - ################################################################################ PYPI_SIMPLE_URL = "https://pypi.org/simple" @@ -190,7 +189,15 @@ class DistributionNotFound(Exception): pass -def download_wheel(name, version, environment, dest_dir=CACHE_THIRDPARTY_DIR, repos=tuple()): +def download_wheel( + name, + version, + environment, + dest_dir=CACHE_THIRDPARTY_DIR, + repos=tuple(), + verbose=False, + echo_func=None, +): """ Download the wheels binary distribution(s) of package ``name`` and ``version`` matching the ``environment`` Environment constraints into the @@ -226,7 +233,11 @@ def download_wheel(name, version, environment, dest_dir=CACHE_THIRDPARTY_DIR, re print( f" download_wheel: Getting wheel from index (or cache): {wheel.download_url}" ) - fetched_wheel_filename = wheel.download(dest_dir=dest_dir) + fetched_wheel_filename = wheel.download( + dest_dir=dest_dir, + verbose=verbose, + echo_func=echo_func, + ) fetched_wheel_filenames.append(fetched_wheel_filename) if fetched_wheel_filenames: @@ -236,7 +247,14 @@ def download_wheel(name, version, environment, dest_dir=CACHE_THIRDPARTY_DIR, re return fetched_wheel_filenames -def download_sdist(name, version, dest_dir=CACHE_THIRDPARTY_DIR, repos=tuple()): +def download_sdist( + name, + version, + dest_dir=CACHE_THIRDPARTY_DIR, + repos=tuple(), + verbose=False, + echo_func=None, +): """ Download the sdist source distribution of package ``name`` and ``version`` into the ``dest_dir`` directory. Return a fetched filename or None. @@ -267,7 +285,11 @@ def download_sdist(name, version, dest_dir=CACHE_THIRDPARTY_DIR, repos=tuple()): if TRACE_DEEP: print(f" download_sdist: Getting sdist from index (or cache): {sdist.download_url}") - fetched_sdist_filename = package.sdist.download(dest_dir=dest_dir) + fetched_sdist_filename = package.sdist.download( + dest_dir=dest_dir, + verbose=verbose, + echo_func=echo_func, + ) if fetched_sdist_filename: # do not futher fetch from other repos if we find in first, typically PyPI @@ -522,7 +544,12 @@ def get_best_download_url(self, repos=tuple()): f" get_best_download_url: {self.filename} not found in {repo.index_url}" ) - def download(self, dest_dir=CACHE_THIRDPARTY_DIR): + def download( + self, + dest_dir=CACHE_THIRDPARTY_DIR, + verbose=False, + echo_func=None, + ): """ Download this distribution into `dest_dir` directory. Return the fetched filename. @@ -540,6 +567,8 @@ def download(self, dest_dir=CACHE_THIRDPARTY_DIR): dest_dir=dest_dir, filename=self.filename, as_text=False, + verbose=verbose, + echo_func=echo_func, ) return self.filename @@ -1024,7 +1053,7 @@ def is_pure(self): def is_pure_wheel(filename): try: return Wheel.from_filename(filename).is_pure() - except: + except Exception: return False @@ -1358,7 +1387,12 @@ class PypiSimpleRepository: repr=False, ) - def _get_package_versions_map(self, name): + def _get_package_versions_map( + self, + name, + verbose=False, + echo_func=None, + ): """ Return a mapping of all available PypiPackage version for this package name. The mapping may be empty. It is ordered by version from oldest to newest @@ -1369,8 +1403,12 @@ def _get_package_versions_map(self, name): if not versions and normalized_name not in self.fetched_package_normalized_names: self.fetched_package_normalized_names.add(normalized_name) try: - links = self.fetch_links(normalized_name=normalized_name) - # note that thsi is sorted so the mapping is also sorted + links = self.fetch_links( + normalized_name=normalized_name, + verbose=verbose, + echo_func=echo_func, + ) + # note that this is sorted so the mapping is also sorted versions = { package.version: package for package in PypiPackage.packages_from_many_paths_or_urls(paths_or_urls=links) @@ -1385,27 +1423,59 @@ def _get_package_versions_map(self, name): return versions - def get_package_versions(self, name): + def get_package_versions( + self, + name, + verbose=False, + echo_func=None, + ): """ Return a mapping of all available PypiPackage version as{version: package} for this package name. The mapping may be empty but not None. It is sorted by version from oldest to newest. """ - return dict(self._get_package_versions_map(name)) + return dict( + self._get_package_versions_map( + name=name, + verbose=verbose, + echo_func=echo_func, + ) + ) - def get_package_version(self, name, version=None): + def get_package_version( + self, + name, + version=None, + verbose=False, + echo_func=None, + ): """ Return the PypiPackage with name and version or None. Return the latest PypiPackage version if version is None. """ if not version: - versions = list(self._get_package_versions_map(name).values()) + versions = list( + self._get_package_versions_map( + name=name, + verbose=verbose, + echo_func=echo_func, + ).values() + ) # return the latest version return versions and versions[-1] else: - return self._get_package_versions_map(name).get(version) - - def fetch_links(self, normalized_name): + return self._get_package_versions_map( + name=name, + verbose=verbose, + echo_func=echo_func, + ).get(version) + + def fetch_links( + self, + normalized_name, + verbose=False, + echo_func=None, + ): """ Return a list of download link URLs found in a PyPI simple index for package name using the `index_url` of this repository. @@ -1415,10 +1485,12 @@ def fetch_links(self, normalized_name): path_or_url=package_url, as_text=True, force=not self.use_cached_index, + verbose=verbose, + echo_func=echo_func, ) links = collect_urls(text) # TODO: keep sha256 - links = [l.partition("#sha256=") for l in links] + links = [lnk.partition("#sha256=") for lnk in links] links = [url for url, _, _sha256 in links] return links @@ -1427,7 +1499,6 @@ def fetch_links(self, normalized_name): DEFAULT_PYPI_REPOS = (PYPI_PUBLIC_REPO,) DEFAULT_PYPI_REPOS_BY_URL = {r.index_url: r for r in DEFAULT_PYPI_REPOS} - ################################################################################ # # Basic file and URL-based operations using a persistent file-based Cache @@ -1447,7 +1518,14 @@ class Cache: def __attrs_post_init__(self): os.makedirs(self.directory, exist_ok=True) - def get(self, path_or_url, as_text=True, force=False): + def get( + self, + path_or_url, + as_text=True, + force=False, + verbose=False, + echo_func=None, + ): """ Return the content fetched from a ``path_or_url`` through the cache. Raise an Exception on errors. Treats the content as text if as_text is @@ -1460,7 +1538,12 @@ def get(self, path_or_url, as_text=True, force=False): if force or not os.path.exists(cached): if TRACE_DEEP: print(f" FILE CACHE MISS: {path_or_url}") - content = get_file_content(path_or_url=path_or_url, as_text=as_text) + content = get_file_content( + path_or_url=path_or_url, + as_text=as_text, + verbose=verbose, + echo_func=echo_func, + ) wmode = "w" if as_text else "wb" with open(cached, wmode) as fo: fo.write(content) @@ -1474,7 +1557,12 @@ def get(self, path_or_url, as_text=True, force=False): CACHE = Cache() -def get_file_content(path_or_url, as_text=True): +def get_file_content( + path_or_url, + as_text=True, + verbose=False, + echo_func=None, +): """ Fetch and return the content at `path_or_url` from either a local path or a remote URL. Return the content as bytes is `as_text` is False. @@ -1482,7 +1570,12 @@ def get_file_content(path_or_url, as_text=True): if path_or_url.startswith("https://"): if TRACE_DEEP: print(f"Fetching: {path_or_url}") - _headers, content = get_remote_file_content(url=path_or_url, as_text=as_text) + _headers, content = get_remote_file_content( + url=path_or_url, + as_text=as_text, + verbose=verbose, + echo_func=echo_func, + ) return content elif path_or_url.startswith("file://") or ( @@ -1517,6 +1610,8 @@ def get_remote_file_content( headers_only=False, headers=None, _delay=0, + verbose=False, + echo_func=None, ): """ Fetch and return a tuple of (headers, content) at `url`. Return content as a @@ -1532,7 +1627,10 @@ def get_remote_file_content( # using a GET with stream=True ensure we get the the final header from # several redirects and that we can ignore content there. A HEAD request may # not get us this last header - print(f" DOWNLOADING: {url}") + if verbose and not echo_func: + echo_func = print + if verbose: + echo_func(f"DOWNLOADING: {url}") with requests.get(url, allow_redirects=True, stream=True, headers=headers) as response: status = response.status_code if status != requests.codes.ok: # NOQA @@ -1561,6 +1659,8 @@ def fetch_and_save( dest_dir, filename, as_text=True, + verbose=False, + echo_func=None, ): """ Fetch content at ``path_or_url`` URL or path and save this to @@ -1571,6 +1671,8 @@ def fetch_and_save( content = CACHE.get( path_or_url=path_or_url, as_text=as_text, + verbose=verbose, + echo_func=echo_func, ) output = os.path.join(dest_dir, filename) wmode = "w" if as_text else "wb" diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 00000000..32517a28 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,14 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# ScanCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/python-inspector for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + + +def pytest_configure(config): + config.addinivalue_line("markers", "online: mark test as requiring network connectivity") diff --git a/tests/data/default-url-expected.json b/tests/data/default-url-expected.json new file mode 100644 index 00000000..aff991a1 --- /dev/null +++ b/tests/data/default-url-expected.json @@ -0,0 +1,34 @@ +{ + "headers": { + "tool_name": "dad", + "tool_homepageurl": "https://github.com/nexB/python-inspector", + "tool_version": "0.5.0", + "options": [ + "--specifier zipp==3.8.0", + "--index-url https://pypi.org/simple", + "--python-version 38", + "--operating-system linux", + "--json " + ], + "notice": "Dependency tree generated with python-inspector.\npython-inspector is a free software tool from nexB Inc. and others.\nVisit https://github.com/nexB/scancode-toolkit/ for support and download.", + "warnings": [], + "errors": [] + }, + "requirements": [ + { + "purl": "pkg:pypi/zipp@3.8.0", + "extracted_requirement": "zipp==3.8.0", + "scope": "install", + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "resolved_package": {} + } + ], + "resolved_dependencies": [ + { + "package": "pkg:pypi/zipp@3.8.0", + "dependencies": [] + } + ] +} \ No newline at end of file diff --git a/tests/data/pinned-requirements.txt b/tests/data/pinned-requirements.txt new file mode 100644 index 00000000..cb292414 --- /dev/null +++ b/tests/data/pinned-requirements.txt @@ -0,0 +1,27 @@ +attrs==21.4.0 +beautifulsoup4==4.11.1 +certifi==2022.5.18.1 +charset-normalizer==2.0.12 +click==8.0.4 +colorama==0.4.4 +commoncode==30.2.0 +dparse2==0.6.1 +idna==3.3 +importlib-metadata==4.8.3 +intbitset==3.0.1 +packageurl-python==0.9.9 +packaging==21.3 +pip-requirements-parser==31.2.0 +pkginfo2==30.0.0 +pyparsing==3.0.9 +PyYAML==6.0 +requests==2.27.1 +resolvelib==0.8.1 +saneyaml==0.5.2 +soupsieve==2.3.2.post1 +text-unidecode==1.3 +toml==0.10.2 +typing==3.6.6 +typing_extensions==4.1.1 +urllib3==1.26.9 +zipp==3.6.0 diff --git a/tests/data/pinned-requirements.txt-expected.json b/tests/data/pinned-requirements.txt-expected.json new file mode 100644 index 00000000..ffb04c91 --- /dev/null +++ b/tests/data/pinned-requirements.txt-expected.json @@ -0,0 +1,398 @@ +{ + "headers": { + "tool_name": "dad", + "tool_homepageurl": "https://github.com/nexB/python-inspector", + "tool_version": "0.5.0", + "options": [ + "--index-url https://pypi.org/simple", + "--python-version 38", + "--operating-system linux", + "--json " + ], + "notice": "Dependency tree generated with python-inspector.\npython-inspector is a free software tool from nexB Inc. and others.\nVisit https://github.com/nexB/scancode-toolkit/ for support and download.", + "warnings": [], + "errors": [] + }, + "requirements": [ + { + "purl": "pkg:pypi/attrs@21.4.0", + "extracted_requirement": "attrs==21.4.0", + "scope": "install", + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "resolved_package": {} + }, + { + "purl": "pkg:pypi/beautifulsoup4@4.11.1", + "extracted_requirement": "beautifulsoup4==4.11.1", + "scope": "install", + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "resolved_package": {} + }, + { + "purl": "pkg:pypi/certifi@2022.5.18.1", + "extracted_requirement": "certifi==2022.5.18.1", + "scope": "install", + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "resolved_package": {} + }, + { + "purl": "pkg:pypi/charset-normalizer@2.0.12", + "extracted_requirement": "charset-normalizer==2.0.12", + "scope": "install", + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "resolved_package": {} + }, + { + "purl": "pkg:pypi/click@8.0.4", + "extracted_requirement": "click==8.0.4", + "scope": "install", + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "resolved_package": {} + }, + { + "purl": "pkg:pypi/colorama@0.4.4", + "extracted_requirement": "colorama==0.4.4", + "scope": "install", + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "resolved_package": {} + }, + { + "purl": "pkg:pypi/commoncode@30.2.0", + "extracted_requirement": "commoncode==30.2.0", + "scope": "install", + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "resolved_package": {} + }, + { + "purl": "pkg:pypi/dparse2@0.6.1", + "extracted_requirement": "dparse2==0.6.1", + "scope": "install", + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "resolved_package": {} + }, + { + "purl": "pkg:pypi/idna@3.3", + "extracted_requirement": "idna==3.3", + "scope": "install", + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "resolved_package": {} + }, + { + "purl": "pkg:pypi/importlib-metadata@4.8.3", + "extracted_requirement": "importlib-metadata==4.8.3", + "scope": "install", + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "resolved_package": {} + }, + { + "purl": "pkg:pypi/intbitset@3.0.1", + "extracted_requirement": "intbitset==3.0.1", + "scope": "install", + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "resolved_package": {} + }, + { + "purl": "pkg:pypi/packageurl-python@0.9.9", + "extracted_requirement": "packageurl-python==0.9.9", + "scope": "install", + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "resolved_package": {} + }, + { + "purl": "pkg:pypi/packaging@21.3", + "extracted_requirement": "packaging==21.3", + "scope": "install", + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "resolved_package": {} + }, + { + "purl": "pkg:pypi/pip-requirements-parser@31.2.0", + "extracted_requirement": "pip-requirements-parser==31.2.0", + "scope": "install", + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "resolved_package": {} + }, + { + "purl": "pkg:pypi/pkginfo2@30.0.0", + "extracted_requirement": "pkginfo2==30.0.0", + "scope": "install", + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "resolved_package": {} + }, + { + "purl": "pkg:pypi/pyparsing@3.0.9", + "extracted_requirement": "pyparsing==3.0.9", + "scope": "install", + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "resolved_package": {} + }, + { + "purl": "pkg:pypi/pyyaml@6.0", + "extracted_requirement": "PyYAML==6.0", + "scope": "install", + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "resolved_package": {} + }, + { + "purl": "pkg:pypi/requests@2.27.1", + "extracted_requirement": "requests==2.27.1", + "scope": "install", + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "resolved_package": {} + }, + { + "purl": "pkg:pypi/resolvelib@0.8.1", + "extracted_requirement": "resolvelib==0.8.1", + "scope": "install", + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "resolved_package": {} + }, + { + "purl": "pkg:pypi/saneyaml@0.5.2", + "extracted_requirement": "saneyaml==0.5.2", + "scope": "install", + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "resolved_package": {} + }, + { + "purl": "pkg:pypi/soupsieve@2.3.2.post1", + "extracted_requirement": "soupsieve==2.3.2.post1", + "scope": "install", + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "resolved_package": {} + }, + { + "purl": "pkg:pypi/text-unidecode@1.3", + "extracted_requirement": "text-unidecode==1.3", + "scope": "install", + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "resolved_package": {} + }, + { + "purl": "pkg:pypi/toml@0.10.2", + "extracted_requirement": "toml==0.10.2", + "scope": "install", + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "resolved_package": {} + }, + { + "purl": "pkg:pypi/typing@3.6.6", + "extracted_requirement": "typing==3.6.6", + "scope": "install", + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "resolved_package": {} + }, + { + "purl": "pkg:pypi/typing-extensions@4.1.1", + "extracted_requirement": "typing_extensions==4.1.1", + "scope": "install", + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "resolved_package": {} + }, + { + "purl": "pkg:pypi/urllib3@1.26.9", + "extracted_requirement": "urllib3==1.26.9", + "scope": "install", + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "resolved_package": {} + }, + { + "purl": "pkg:pypi/zipp@3.6.0", + "extracted_requirement": "zipp==3.6.0", + "scope": "install", + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "resolved_package": {} + } + ], + "resolved_dependencies": [ + { + "package": "pkg:pypi/attrs@21.4.0", + "dependencies": [] + }, + { + "package": "pkg:pypi/beautifulsoup4@4.11.1", + "dependencies": [ + "pkg:pypi/soupsieve@2.3.2.post1" + ] + }, + { + "package": "pkg:pypi/certifi@2022.5.18.1", + "dependencies": [] + }, + { + "package": "pkg:pypi/charset-normalizer@2.0.12", + "dependencies": [] + }, + { + "package": "pkg:pypi/click@8.0.4", + "dependencies": [] + }, + { + "package": "pkg:pypi/colorama@0.4.4", + "dependencies": [] + }, + { + "package": "pkg:pypi/commoncode@30.2.0", + "dependencies": [ + "pkg:pypi/attrs@21.4.0", + "pkg:pypi/beautifulsoup4@4.11.1", + "pkg:pypi/click@8.0.4", + "pkg:pypi/intbitset@3.0.1", + "pkg:pypi/requests@2.27.1", + "pkg:pypi/saneyaml@0.5.2", + "pkg:pypi/text-unidecode@1.3" + ] + }, + { + "package": "pkg:pypi/dparse2@0.6.1", + "dependencies": [ + "pkg:pypi/packaging@21.3", + "pkg:pypi/pyyaml@6.0", + "pkg:pypi/toml@0.10.2" + ] + }, + { + "package": "pkg:pypi/idna@3.3", + "dependencies": [] + }, + { + "package": "pkg:pypi/importlib-metadata@4.8.3", + "dependencies": [ + "pkg:pypi/zipp@3.6.0" + ] + }, + { + "package": "pkg:pypi/intbitset@3.0.1", + "dependencies": [] + }, + { + "package": "pkg:pypi/packageurl-python@0.9.9", + "dependencies": [] + }, + { + "package": "pkg:pypi/packaging@21.3", + "dependencies": [ + "pkg:pypi/pyparsing@3.0.9" + ] + }, + { + "package": "pkg:pypi/pip-requirements-parser@31.2.0", + "dependencies": [ + "pkg:pypi/packaging@21.3" + ] + }, + { + "package": "pkg:pypi/pkginfo2@30.0.0", + "dependencies": [] + }, + { + "package": "pkg:pypi/pyparsing@3.0.9", + "dependencies": [] + }, + { + "package": "pkg:pypi/pyyaml@6.0", + "dependencies": [] + }, + { + "package": "pkg:pypi/requests@2.27.1", + "dependencies": [ + "pkg:pypi/certifi@2022.5.18.1", + "pkg:pypi/charset-normalizer@2.0.12", + "pkg:pypi/idna@3.3", + "pkg:pypi/urllib3@1.26.9" + ] + }, + { + "package": "pkg:pypi/resolvelib@0.8.1", + "dependencies": [] + }, + { + "package": "pkg:pypi/saneyaml@0.5.2", + "dependencies": [ + "pkg:pypi/pyyaml@6.0" + ] + }, + { + "package": "pkg:pypi/soupsieve@2.3.2.post1", + "dependencies": [] + }, + { + "package": "pkg:pypi/text-unidecode@1.3", + "dependencies": [] + }, + { + "package": "pkg:pypi/toml@0.10.2", + "dependencies": [] + }, + { + "package": "pkg:pypi/typing-extensions@4.1.1", + "dependencies": [] + }, + { + "package": "pkg:pypi/typing@3.6.6", + "dependencies": [] + }, + { + "package": "pkg:pypi/urllib3@1.26.9", + "dependencies": [] + }, + { + "package": "pkg:pypi/zipp@3.6.0", + "dependencies": [] + } + ] +} \ No newline at end of file diff --git a/tests/data/single-url-expected.json b/tests/data/single-url-expected.json new file mode 100644 index 00000000..aff991a1 --- /dev/null +++ b/tests/data/single-url-expected.json @@ -0,0 +1,34 @@ +{ + "headers": { + "tool_name": "dad", + "tool_homepageurl": "https://github.com/nexB/python-inspector", + "tool_version": "0.5.0", + "options": [ + "--specifier zipp==3.8.0", + "--index-url https://pypi.org/simple", + "--python-version 38", + "--operating-system linux", + "--json " + ], + "notice": "Dependency tree generated with python-inspector.\npython-inspector is a free software tool from nexB Inc. and others.\nVisit https://github.com/nexB/scancode-toolkit/ for support and download.", + "warnings": [], + "errors": [] + }, + "requirements": [ + { + "purl": "pkg:pypi/zipp@3.8.0", + "extracted_requirement": "zipp==3.8.0", + "scope": "install", + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "resolved_package": {} + } + ], + "resolved_dependencies": [ + { + "package": "pkg:pypi/zipp@3.8.0", + "dependencies": [] + } + ] +} \ No newline at end of file diff --git a/tests/data/tilde_req-expected.json b/tests/data/tilde_req-expected.json new file mode 100644 index 00000000..7809e80f --- /dev/null +++ b/tests/data/tilde_req-expected.json @@ -0,0 +1,35 @@ +{ + "headers": { + "tool_name": "dad", + "tool_homepageurl": "https://github.com/nexB/python-inspector", + "tool_version": "0.5.0", + "options": [ + "--specifier zipp~=3.8.0", + "--index-url https://pypi.org/simple", + "--index-url https://thirdparty.aboutcode.org/pypi/simple/", + "--python-version 38", + "--operating-system linux", + "--json " + ], + "notice": "Dependency tree generated with python-inspector.\npython-inspector is a free software tool from nexB Inc. and others.\nVisit https://github.com/nexB/scancode-toolkit/ for support and download.", + "warnings": [], + "errors": [] + }, + "requirements": [ + { + "purl": "pkg:pypi/zipp", + "extracted_requirement": "zipp~=3.8.0", + "scope": "install", + "is_runtime": true, + "is_optional": false, + "is_resolved": false, + "resolved_package": {} + } + ], + "resolved_dependencies": [ + { + "package": "pkg:pypi/zipp@3.8.0", + "dependencies": [] + } + ] +} \ No newline at end of file diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 00000000..508a5481 --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,186 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# ScanCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/python-inspector for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +import json +import os + +import pytest +from click.testing import CliRunner +from commoncode.testcase import FileDrivenTesting + +from python_inspector.resolve_cli import resolve_dependencies + +# Used for tests to regenerate fixtures with regen=True +REGEN_TEST_FIXTURES = os.getenv("PYINSP_REGEN_TEST_FIXTURES", False) + +test_env = FileDrivenTesting() +test_env.test_data_dir = os.path.join(os.path.dirname(__file__), "data") + + +@pytest.mark.online +def test_cli_with_default_urls(): + expected_file = test_env.get_test_loc("default-url-expected.json", must_exist=False) + specifier = "zipp==3.8.0" + extra_options = [ + "--use-pypi-json-api", + ] + check_specs_resolution( + specifier=specifier, + expected_file=expected_file, + extra_options=extra_options, + regen=REGEN_TEST_FIXTURES, + ) + + +@pytest.mark.online +def test_cli_with_single_index_url(): + expected_file = test_env.get_test_loc("single-url-expected.json", must_exist=False) + specifier = "zipp==3.8.0" + extra_options = [ + "--index-url", + "https://pypi.org/simple", + ] + check_specs_resolution( + specifier=specifier, + expected_file=expected_file, + extra_options=extra_options, + regen=REGEN_TEST_FIXTURES, + ) + + +@pytest.mark.online +def test_cli_with_multiple_index_url_and_tilde_req(): + expected_file = test_env.get_test_loc("tilde_req-expected.json", must_exist=False) + specifier = "zipp~=3.8.0" + extra_options = [ + "--index-url", + "https://pypi.org/simple", + "--index-url", + "https://thirdparty.aboutcode.org/pypi/simple/", + ] + check_specs_resolution( + specifier=specifier, + expected_file=expected_file, + extra_options=extra_options, + regen=REGEN_TEST_FIXTURES, + ) + + +@pytest.mark.online +def test_cli_with_pinned_requirements_file(): + requirements_file = test_env.get_test_loc("pinned-requirements.txt") + expected_file = test_env.get_test_loc("pinned-requirements.txt-expected.json", must_exist=False) + check_requirements_resolution( + requirements_file=requirements_file, + expected_file=expected_file, + regen=REGEN_TEST_FIXTURES, + ) + + +def check_specs_resolution( + specifier, + expected_file, + extra_options=tuple(), + regen=REGEN_TEST_FIXTURES, +): + result_file = test_env.get_temp_file("json") + options = ["--specifier", specifier, "--json", result_file] + options.extend(extra_options) + run_cli(options=options) + check_json_results( + result_file=result_file, + expected_file=expected_file, + regen=regen, + ) + + +def check_requirements_resolution( + requirements_file, + expected_file, + extra_options=tuple(), + regen=REGEN_TEST_FIXTURES, +): + result_file = test_env.get_temp_file("json") + options = ["--requirement", requirements_file, "--json", result_file] + options.extend(extra_options) + run_cli(options=options) + check_json_results( + result_file=result_file, + expected_file=expected_file, + regen=regen, + ) + + +def check_json_results(result_file, expected_file, clean=True, regen=REGEN_TEST_FIXTURES): + """ + Check the ``result_file`` JSON results against the ``expected_file`` + expected JSON results. + + If ``clean`` is True, remove headers data that can change across runs to + provide stable test resultys. + + If ``regen`` is True the expected_file WILL BE overwritten with the new + results from ``results_file``. This is convenient for updating tests + expectations. + """ + with open(result_file) as res: + results = json.load(res) + + if clean: + clean_results(results) + + if regen: + with open(expected_file, "w") as reg: + json.dump(results, reg, indent=2, separators=(",", ": ")) + expected = results + else: + with open(expected_file) as res: + expected = json.load(res) + + if clean: + clean_results(expected) + + assert results == expected + + +def clean_results(results): + """ + Return cleaned results removing transient values that can change across test + runs. + """ + headers = results.get("headers", {}) + options = headers.get("options", []) + headers["options"] = [o for o in options if not o.startswith("--requirement")] + return results + + +def run_cli(options, cli=resolve_dependencies, expected_rc=0, env=None): + """ + Run a command line resolution. Return a click.testing.Result object. + """ + + if not env: + env = dict(os.environ) + + runner = CliRunner() + result = runner.invoke(cli, options, catch_exceptions=False, env=env) + + if result.exit_code != expected_rc: + output = result.output + error = f""" +Failure to run: +rc: {result.exit_code} +python-inspector {options} +output: +{output} +""" + assert result.exit_code == expected_rc, error + return result diff --git a/tests/test_resolution.py b/tests/test_resolution.py new file mode 100644 index 00000000..52fb6d97 --- /dev/null +++ b/tests/test_resolution.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# ScanCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/python-inspector for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# +import packaging +import pytest +from packaging.requirements import Requirement + +from python_inspector.resolution import get_resolved_dependencies +from python_inspector.resolution import is_valid_version +from python_inspector.utils_pypi import PYPI_PUBLIC_REPO +from python_inspector.utils_pypi import Environment + + +@pytest.mark.online +def test_get_resolved_dependencies_with_flask_and_python_310(): + req = [Requirement("flask==2.1.2")] + results = get_resolved_dependencies( + requirements=req, + environment=Environment( + python_version="310", + operating_system="linux", + ), + repos=[PYPI_PUBLIC_REPO], + as_tree=False, + ) + as_list = [p["package"] for p in results] + assert as_list == [ + "pkg:pypi/click@8.1.3", + "pkg:pypi/flask@2.1.2", + "pkg:pypi/itsdangerous@2.1.2", + "pkg:pypi/jinja2@3.1.2", + "pkg:pypi/markupsafe@2.1.1", + "pkg:pypi/werkzeug@2.1.2", + ] + + +@pytest.mark.online +def test_get_resolved_dependencies_with_flask_and_python_310_windows(): + req = [Requirement("flask==2.1.2")] + results = get_resolved_dependencies( + requirements=req, + environment=Environment( + python_version="310", + operating_system="windows", + ), + repos=[PYPI_PUBLIC_REPO], + as_tree=False, + ) + as_list = [p["package"] for p in results] + assert as_list == [ + "pkg:pypi/click@8.1.3", + "pkg:pypi/colorama@0.4.4", + "pkg:pypi/flask@2.1.2", + "pkg:pypi/itsdangerous@2.1.2", + "pkg:pypi/jinja2@3.1.2", + "pkg:pypi/markupsafe@2.1.1", + "pkg:pypi/werkzeug@2.1.2", + ] + + +@pytest.mark.online +def test_get_resolved_dependencies_with_flask_and_python_36(): + req = [Requirement("flask==2.1.2")] + results = get_resolved_dependencies( + requirements=req, + environment=Environment( + python_version="36", + operating_system="linux", + ), + repos=[PYPI_PUBLIC_REPO], + as_tree=False, + ) + as_list = [p["package"] for p in results] + + assert as_list == [ + "pkg:pypi/click@8.1.3", + "pkg:pypi/flask@2.1.2", + "pkg:pypi/importlib-metadata@4.11.4", + "pkg:pypi/itsdangerous@2.1.2", + "pkg:pypi/jinja2@3.1.2", + "pkg:pypi/markupsafe@2.0.1", + "pkg:pypi/typing-extensions@4.2.0", + "pkg:pypi/werkzeug@2.1.2", + "pkg:pypi/zipp@3.8.0", + ] + + +@pytest.mark.online +def test_get_resolved_dependencies_with_tilde_requirement_using_json_api(): + req = [Requirement("flask~=2.1.2")] + results = get_resolved_dependencies( + requirements=req, + as_tree=False, + environment=Environment( + python_version="38", + operating_system="linux", + ), + ) + as_list = [p["package"] for p in results] + assert as_list == [ + "pkg:pypi/click@8.1.3", + "pkg:pypi/flask@2.1.2", + "pkg:pypi/importlib-metadata@4.11.4", + "pkg:pypi/itsdangerous@2.1.2", + "pkg:pypi/jinja2@3.1.2", + "pkg:pypi/markupsafe@2.1.1", + "pkg:pypi/werkzeug@2.1.2", + "pkg:pypi/zipp@3.8.0", + ] + + +def test_is_valid_version(): + parsed_version = packaging.version.parse("2.1.2") + requirements = {"flask": [Requirement("flask>2.0.0")]} + bad_versions = [] + identifier = "flask" + assert is_valid_version(parsed_version, requirements, identifier, bad_versions) diff --git a/tests/test_utils_pip_compatibility_tags.py b/tests/test_utils_pip_compatibility_tags.py index 5768403f..a492912c 100644 --- a/tests/test_utils_pip_compatibility_tags.py +++ b/tests/test_utils_pip_compatibility_tags.py @@ -95,7 +95,9 @@ def test_manylinux2010_implies_manylinux1(self, manylinux2010, manylinux1): Specifying manylinux2010 implies manylinux1. """ groups = {} - supported = utils_pip_compatibility_tags.get_supported(platforms=[manylinux2010]) + supported = utils_pip_compatibility_tags.get_supported( + platforms=[manylinux2010], + ) for tag in supported: groups.setdefault((tag.interpreter, tag.abi), []).append(tag.platform)