From d93daaa07297c6cd7e2cca0a3f98ea4c1ce1d4db Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Tue, 14 Jun 2022 00:40:55 +0530 Subject: [PATCH 01/19] Add dependency resolution Signed-off-by: Tushar Goel --- src/_packagedcode/models.py | 6 + src/_packagedcode/pypi.py | 2 +- src/python_inspector/resolution.py | 307 +++++++++++++++++++++++ src/python_inspector/resolution.py.ABOUT | 14 ++ src/python_inspector/resolve_cli.py | 28 ++- tests/conftest.py | 14 ++ tests/test_cli.py | 54 ++++ tests/test_resolution.py | 107 ++++++++ 8 files changed, 525 insertions(+), 7 deletions(-) create mode 100644 src/python_inspector/resolution.py create mode 100644 src/python_inspector/resolution.py.ABOUT create mode 100644 tests/conftest.py create mode 100644 tests/test_cli.py create mode 100644 tests/test_resolution.py diff --git a/src/_packagedcode/models.py b/src/_packagedcode/models.py index 11911657..ec4faf72 100644 --- a/src/_packagedcode/models.py +++ b/src/_packagedcode/models.py @@ -356,6 +356,12 @@ class DependentPackage(ModelMixin): "lockfiles for Composer or Cargo contain extra dependency data.", ) + # dependencies = List( + # item_type="DependentPackage", + # label="dependencies", + # help="A list of DependentPackage for this package.", + # ) + @attr.attributes(slots=True) class Dependency(DependentPackage): diff --git a/src/_packagedcode/pypi.py b/src/_packagedcode/pypi.py index 1f73d3c8..9cdef5b2 100644 --- a/src/_packagedcode/pypi.py +++ b/src/_packagedcode/pypi.py @@ -812,7 +812,7 @@ def get_requires_dependencies(requires, default_scope="install"): is_runtime=True, is_optional=False, is_resolved=is_resolved, - extracted_requirement=requirement, + extracted_requirement=str(req), ) ) diff --git a/src/python_inspector/resolution.py b/src/python_inspector/resolution.py new file mode 100644 index 00000000..ade40259 --- /dev/null +++ b/src/python_inspector/resolution.py @@ -0,0 +1,307 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# ScanCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/scancode-toolkit for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +import collections +import operator +import os +from typing import List + +import packaging.markers +import packaging.requirements +import packaging.specifiers +import packaging.utils +import packaging.version +import requests +from packageurl import PackageURL +from packaging.requirements import Requirement +from resolvelib import AbstractProvider +from resolvelib import Resolver +from resolvelib.reporters import BaseReporter + +from _packagedcode.pypi import PypiWheelHandler +from python_inspector.utils_pypi import CACHE_THIRDPARTY_DIR +from python_inspector.utils_pypi import PYPI_PUBLIC_REPO +from python_inspector.utils_pypi import PYPI_SIMPLE_URL +from python_inspector.utils_pypi import Environment +from python_inspector.utils_pypi import PypiSimpleRepository +from python_inspector.utils_pypi import download_wheel + +Candidate = collections.namedtuple("Candidate", "name version extras") + + +def get_response(url): + """ + Return a response for the given url. + """ + resp = requests.get(url) + if resp.status_code == 200: + return resp.json() + return None + + +def is_valid_version(parsed_version, requirements, identifier, bad_versions): + """ + Return True if the parsed_version is valid for the given identifier. + """ + if ( + any(parsed_version not in r.specifier for r in requirements[identifier]) + or parsed_version in bad_versions + ): + return False + return True + + +class PythonInputProvider(AbstractProvider): + def __init__(self, environment=None, repos=[]): + self.environment = environment + self.repos = repos + self.versions_by_package = {} + self.dependencies_by_purl = {} + + def identify(self, requirement_or_candidate): + name = packaging.utils.canonicalize_name(requirement_or_candidate.name) + if requirement_or_candidate.extras: + extras_str = ",".join(sorted(requirement_or_candidate.extras)) + return "{}[{}]".format(name, extras_str) + return name + + def get_preference( + self, + identifier, + resolutions, + candidates, + information, + backtrack_causes, + ): + transitive = all(p is not None for _, p in information[identifier]) + return (transitive, identifier) + + def get_versions_for_package(self, name, repo=None): + """ + Return a list of versions for a package. + """ + versions = [] + if repo and self.environment: + for version, package in repo._get_package_versions_map(name).items(): + wheels = package.get_supported_wheels(environment=self.environment) + if list(wheels): + versions.append(version) + else: + if name not in self.versions_by_package: + api_url = f"https://pypi.org/pypi/{name}/json" + resp = get_response(api_url) + if not resp: + self.versions_by_package[name] = [] + releases = resp.get("releases") or {} + self.versions_by_package[name] = releases.keys() or [] + versions = self.versions_by_package[name] + return versions + + def get_requirements_for_package(self, purl, candidate): + """ + Yield requirements for a package. + """ + if self.repos and self.environment: + wheels = download_wheel( + name=candidate.name, + version=str(candidate.version), + environment=self.environment, + repos=self.repos, + ) + for wheel in wheels: + deps = list(PypiWheelHandler.parse(os.path.join(CACHE_THIRDPARTY_DIR, wheel))) + assert len(deps) == 1 + deps = deps[0].dependencies + for dep in deps: + if dep.scope == "install": + yield packaging.requirements.Requirement(str(dep.extracted_requirement)) + else: + if str(purl) not in self.dependencies_by_purl: + api_url = f"https://pypi.org/pypi/{purl.name}/{purl.version}/json" + resp = get_response(api_url) + if not resp: + self.dependencies_by_purl[str(purl)] = [] + info = resp.get("info") or {} + requires_dist = info.get("requires_dist") or [] + self.dependencies_by_purl[str(purl)] = requires_dist + for dependency in self.dependencies_by_purl[str(purl)]: + yield packaging.requirements.Requirement(dependency) + + def get_candidates(self, all_versions, requirements, identifier, bad_versions, name, extras): + """ + Generate candidates for the given identifier. + """ + for version in all_versions: + parsed_version = packaging.version.parse(version) + if not is_valid_version(parsed_version, requirements, identifier, bad_versions): + continue + yield Candidate(name=name, version=parsed_version, extras=extras) + + def _iter_matches(self, identifier, requirements, incompatibilities): + """ + Return a list of candidates for the given identifier. + """ + name, _, _ = identifier.partition("[") + bad_versions = {c.version for c in incompatibilities[identifier]} + extras = {e for r in requirements[identifier] for e in r.extras} + if not self.repos: + all_versions = self.get_versions_for_package(name) + yield from self.get_candidates( + all_versions, requirements, identifier, bad_versions, name, extras + ) + else: + for repo in self.repos: + all_versions = self.get_versions_for_package(name, repo) + yield from self.get_candidates( + all_versions, requirements, identifier, bad_versions, name, extras + ) + + def find_matches(self, identifier, requirements, incompatibilities): + candidates = sorted( + self._iter_matches(identifier, requirements, incompatibilities), + key=operator.attrgetter("version"), + reverse=True, + ) + return candidates + + def is_satisfied_by(self, requirement, candidate): + return candidate.version in requirement.specifier + + def _iter_dependencies(self, candidate): + """ + Yield dependencies for the given candidate. + """ + name = packaging.utils.canonicalize_name(candidate.name) + if candidate.extras: + r = f"{name}=={candidate.version}" + yield packaging.requirements.Requirement(r) + purl = PackageURL( + type="pypi", + name=name, + version=str(candidate.version), + ) + for r in self.get_requirements_for_package(purl, candidate): + if r.marker is None: + yield r + else: + if r.marker.evaluate({"extra": ""}): + yield r + + def get_dependencies(self, candidate): + return list(self._iter_dependencies(candidate)) + + +def get_all_srcs(mapping, graph): + """ + Return a list of all sources in the graph. + """ + for name in mapping.keys(): + if list(graph.iter_parents(name)) == [None]: + yield name + + +def dfs(mapping, graph, src): + """ + Return a recursive mapping of dependencies. + """ + children = list(graph.iter_children(src)) + src_purl = PackageURL( + type="pypi", + name=src, + version=str(mapping[src].version), + ) + if not children: + return dict(package=str(src_purl), dependencies=[]) + + return dict( + package=str(src_purl), + dependencies=sorted([dfs(mapping, graph, c) for c in children], key=lambda d: d["package"]), + ) + + +def format_resolution(result): + """ + Return a formatted resolution. + """ + mapping = result.mapping + graph = result.graph + as_list = [ + str( + PackageURL( + type="pypi", + name=name, + version=str(candidate.version), + ) + ) + for name, candidate in mapping.items() + ] + + as_parent_children = [] + parents = mapping.keys() + for parent in parents: + parent_purl = PackageURL( + type="pypi", + name=parent, + version=str(mapping[parent].version), + ) + dependencies = [] + for dependency in graph.iter_children(parent): + dep_purl = PackageURL( + type="pypi", + name=dependency, + version=str(mapping[dependency].version), + ) + dependencies.append(str(dep_purl)) + dependencies.sort() + parent_children = dict(package=str(parent_purl), dependencies=dependencies) + as_parent_children.append(parent_children) + + srcs = list(get_all_srcs(mapping=mapping, graph=graph)) + dependencies = [] + for src in srcs: + dependencies.append(dfs(mapping=mapping, graph=graph, src=src)) + + as_list.sort() + as_parent_children.sort(key=lambda d: d["package"]) + dependencies.sort(key=lambda d: d["package"]) + as_tree = dict(dependencies=dependencies) + return as_list, as_parent_children, as_tree + + +def pypi_simple_repo_in_repos(repos: PypiSimpleRepository): + """ + Return True if simple pypi index_url is present in any of the repos + """ + for repo in repos: + if repo.index_url == PYPI_SIMPLE_URL: + return True + return False + + +def resolution( + requirements: List[Requirement], + environment: Environment = None, + repos: List[PypiSimpleRepository] = [], + return_as_parent_children: bool = True, + return_as_tree: bool = False, + return_as_list: bool = False, +): + """ + Return a resolution for the given requirements. + """ + if repos and not pypi_simple_repo_in_repos(repos): + repos.append(PYPI_PUBLIC_REPO) + resolver = Resolver(PythonInputProvider(environment, repos), BaseReporter()) + as_list, as_parent_children, as_tree = format_resolution(resolver.resolve(requirements)) + if return_as_parent_children: + return as_parent_children + if return_as_tree: + return as_tree + if return_as_list: + return as_list diff --git a/src/python_inspector/resolution.py.ABOUT b/src/python_inspector/resolution.py.ABOUT new file mode 100644 index 00000000..2b9f0090 --- /dev/null +++ b/src/python_inspector/resolution.py.ABOUT @@ -0,0 +1,14 @@ +about_resource: resolution.py +package_url: pkg:github.com/sarugaku/resolvelib/@a5ae68140afac49dd1a1a8e87eff9550db4a586b#tests/functional/python/test_resolvers_python.py +type: github +namespace: sarugaku +name: resolvelib +version: a5ae68140afac49dd1a1a8e87eff9550db4a586b +subpath: tests/functional/python/test_resolvers_python.py + +download_url: https://github.com/sarugaku/resolvelib/blob/a5ae68140afac49dd1a1a8e87eff9550db4a586b/tests/functional/python/test_resolvers_python.py +copyright: Copyright (c) 2018, Tzu-ping Chung +license_expression: isc +notes: The PythonInputProvider is copied and heavily modified from + resolvelib tests/functional/python/test_resolvers_python.py + \ No newline at end of file diff --git a/src/python_inspector/resolve_cli.py b/src/python_inspector/resolve_cli.py index f7363541..610cee1e 100644 --- a/src/python_inspector/resolve_cli.py +++ b/src/python_inspector/resolve_cli.py @@ -13,10 +13,12 @@ import sys import click +from packaging.requirements import Requirement from python_inspector import dependencies from python_inspector import utils_pypi from python_inspector.cli_utils import FileOptionType +from python_inspector.resolution import resolution TRACE = False @@ -69,7 +71,6 @@ "index_urls", type=str, metavar="INDEX", - default=utils_pypi.PYPI_INDEX_URLS, show_default=True, multiple=True, help="PyPI index URL(s) to use in order of preference. " @@ -116,6 +117,19 @@ def resolve_dependencies( Download from the provided PyPI simple --index-url INDEX(s) URLs. Error and progress are printed to stderr. + Default environment is the Python version - 3.8 and OS - linux. + + 1) If no index_url is provided, the PyPI JSON API is used and environment will be ignored in that case. + + For example: + dad --spec "flask==2.1.2" --json - + + 2) If an index_url is provided, the environment will be used to resolve the dependencies. + (If no environment is provided default environment will be used.) + + For example: + dad --spec "flask==2.1.2" --index-url https://pypi.org/simple --json - + For example:: dad --spec "flask" --requirement etc/scripts/requirements.txt --json - """ @@ -172,26 +186,28 @@ def resolve_dependencies( print(" ", repo) # resolve dependencies proper - resolved_dependencies = resolve(direct_dependencies) + resolved_dependencies = resolve(direct_dependencies, environment, repos) write_output(results=resolved_dependencies, json_output=json_output) if debug: print("done!") -def resolve(direct_dependencies): +def resolve(direct_dependencies, environment, repos): """ Resolve dependencies given a ``direct_dependencies`` list of DependentPackage and return SOMETHING TBD. """ - from packaging.requirements import Requirement reqs = [Requirement(d.extracted_requirement) for d in direct_dependencies] + as_parent_children = resolution(reqs, environment, repos) return dict( headers=[dict(tool="dad")], - dependencies=[d.to_dict() for d in direct_dependencies], - requirements=[str(r) for r in reqs], + requirements=[d.to_dict() for d in direct_dependencies], + resolved_dependencies=dict( + as_parent_children=as_parent_children, + ), ) diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 00000000..32517a28 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,14 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# ScanCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/python-inspector for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + + +def pytest_configure(config): + config.addinivalue_line("markers", "online: mark test as requiring network connectivity") diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 00000000..13f9f321 --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# ScanCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/python-inspector for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +import pytest +from click.testing import CliRunner + +from python_inspector.resolve_cli import resolve_dependencies as cli + + +@pytest.mark.online +def test_cli_with_json_api(): + runner = CliRunner() + result = runner.invoke( + cli, + ["--spec", "zipp==3.8.0", "--json", "-"], + ) + assert result.exit_code == 0 + + +@pytest.mark.online +def test_cli_with_single_index_url(): + runner = CliRunner() + result = runner.invoke( + cli, + ["--spec", "zipp==3.8.0", "--index-url", "https://pypi.org/simple", "--json", "-"], + ) + assert result.exit_code == 0 + + +@pytest.mark.online +def test_cli_with_multiple_index_url_and_tilde_req(): + runner = CliRunner() + result = runner.invoke( + cli, + [ + "--spec", + "zipp~=3.8.0", + "--index-url", + "https://pypi.org/simple", + "--index-url", + "https://thirdparty.aboutcode.org/pypi/simple/", + "--json", + "-", + ], + ) + assert result.exit_code == 0 diff --git a/tests/test_resolution.py b/tests/test_resolution.py new file mode 100644 index 00000000..275921be --- /dev/null +++ b/tests/test_resolution.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# ScanCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/python-inspector for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# +import packaging +import pytest +from packaging.requirements import Requirement + +from python_inspector.resolution import is_valid_version +from python_inspector.resolution import pypi_simple_repo_in_repos +from python_inspector.resolution import resolution +from python_inspector.utils_pypi import PYPI_PUBLIC_REPO +from python_inspector.utils_pypi import Environment + + +@pytest.mark.online +def test_resolvelib_with_flask_and_python_310(): + req = [Requirement("flask==2.1.2")] + as_list = resolution( + requirements=req, + environment=Environment( + python_version="310", + operating_system="linux", + ), + repos=[PYPI_PUBLIC_REPO], + return_as_parent_children=False, + return_as_list=True, + ) + assert as_list == [ + "pkg:pypi/click@8.1.3", + "pkg:pypi/flask@2.1.2", + "pkg:pypi/importlib-metadata@4.11.4", + "pkg:pypi/itsdangerous@2.1.2", + "pkg:pypi/jinja2@3.1.2", + "pkg:pypi/markupsafe@2.1.1", + "pkg:pypi/werkzeug@2.1.2", + "pkg:pypi/zipp@3.8.0", + ] + + +@pytest.mark.online +def test_resolvelib_with_flask_and_python_36(): + req = [Requirement("flask==2.1.2")] + as_list = resolution( + requirements=req, + environment=Environment( + python_version="36", + operating_system="linux", + ), + repos=[PYPI_PUBLIC_REPO], + return_as_parent_children=False, + return_as_list=True, + ) + + assert as_list == [ + "pkg:pypi/click@8.1.3", + "pkg:pypi/flask@2.1.2", + "pkg:pypi/importlib-metadata@4.11.4", + "pkg:pypi/itsdangerous@2.1.2", + "pkg:pypi/jinja2@3.1.2", + "pkg:pypi/markupsafe@2.0.1", + "pkg:pypi/werkzeug@2.1.2", + "pkg:pypi/zipp@3.8.0", + ] + + +@pytest.mark.online +def test_resolvelib_with_tilde_requirement_using_json_api(): + req = [Requirement("flask~=2.1.2")] + as_list = resolution( + requirements=req, + return_as_parent_children=False, + return_as_list=True, + ) + + assert as_list == [ + "pkg:pypi/click@8.1.3", + "pkg:pypi/flask@2.1.2", + "pkg:pypi/importlib-metadata@4.11.4", + "pkg:pypi/itsdangerous@2.1.2", + "pkg:pypi/jinja2@3.1.2", + "pkg:pypi/markupsafe@2.1.1", + "pkg:pypi/werkzeug@2.1.2", + "pkg:pypi/zipp@3.8.0", + ] + + +def test_pypi_simple_repo_in_repos(): + assert pypi_simple_repo_in_repos(repos=[PYPI_PUBLIC_REPO]) == True + + +def test_pypi_simple_repo_in_repos_not_present(): + assert pypi_simple_repo_in_repos(repos=[]) == False + + +def test_is_valid_version(): + parsed_version = packaging.version.parse("2.1.2") + requirements = {"flask": [Requirement("flask>2.0.0")]} + bad_versions = [] + identifier = "flask" + assert is_valid_version(parsed_version, requirements, identifier, bad_versions) == True From 8b448f452569de6ad95c764776892f0891267bc4 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Tue, 14 Jun 2022 10:07:23 +0200 Subject: [PATCH 02/19] Use correct ReST syntax Signed-off-by: Philippe Ombredanne --- docs/source/dependencies-design.rst | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/source/dependencies-design.rst b/docs/source/dependencies-design.rst index 7b0c9d63..952f3817 100644 --- a/docs/source/dependencies-design.rst +++ b/docs/source/dependencies-design.rst @@ -21,11 +21,11 @@ For each dependency, there can be a version "requirement" that can be an exact version, a version expression (aka. version specifier) as defined in `pep-0440 `__ with additional OS and environment tags and constraints as specified in -`pep-0508 `__ . +`pep-0508 `__ . -In particular the required Python version of a package (or version specifier) -can be set for the whole package (with the``python_requires`` attribute) or as -a marker for a given direct or indirect dependency. pip processes requirement +In particular the required Python version of a package (or version specifier) +can be set for the whole package (with the``python_requires`` attribute) or as +a marker for a given direct or indirect dependency. pip processes requirement specifiers and constraints from a "requirements" file and internally resolves dependency versions recursively by querying the PyPI Python package index repository at https://PyPI.org @@ -129,7 +129,7 @@ And a flat list of unique dependencies would be: - thing 3.0 - shebang 1.0 -The implementation may likely be similar to +The implementation may likely be similar to `pipgrip `__, but using the `resolvelib `__ library as used and vendored in pip instead of an implementation of @@ -139,7 +139,7 @@ The expected benefit of this tool is a simpler way to resolve Python dependencies that will not require complex installation of a Python toolchain specific to a given project environment when the goal is only to resolve dependencies. In particular the key new capability is to run this tool on a -single Python version and resolve versions for alternative Python versions, +single Python version and resolve versions for alternative Python versions, operating systems and architectures without having to install all the packages in the dependency tree. @@ -158,7 +158,8 @@ The outline of the processing is to: - For each top-level requirement (e.g. name/version): - Fetch all the corresponding versions metadata using the PyPI API(s) - - Fetch the packages as needed to further obtain the next-level dependencies, and this recursively + - Fetch the packages as needed to further obtain the next-level + dependencies, and this recursively - Resolve a correct dependency version for each name. - Dump JSON @@ -168,7 +169,7 @@ User experience: ---------------- The goal of the command line interface and user experience is to be -obvious and familiar to a pip user. +obvious and familiar to a pip user. Create a new CLI named "dad" short for "dad analyzes dependencies" with these key options: @@ -352,4 +353,3 @@ ScanCode Toolkit can detect the and normalize the declared licenses in package metadata and also collect and normalize all the metadata. This could be a refinement for later. - From b8a45ed4b866612450be14ee30da1f5a1e58d2ad Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Tue, 14 Jun 2022 10:12:33 +0200 Subject: [PATCH 03/19] Run CI on Python 3.8 and up Signed-off-by: Philippe Ombredanne --- azure-pipelines.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 6ca19c4d..e67d2e14 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -11,7 +11,7 @@ jobs: parameters: job_name: ubuntu18_cpython image_name: ubuntu-18.04 - python_versions: ['3.6', '3.7', '3.8', '3.9', '3.10'] + python_versions: ['3.8', '3.9', '3.10'] test_suites: all: venv/bin/pytest -n 2 -vvs @@ -19,7 +19,7 @@ jobs: parameters: job_name: ubuntu20_cpython image_name: ubuntu-20.04 - python_versions: ['3.6', '3.7', '3.8', '3.9', '3.10'] + python_versions: ['3.8', '3.9', '3.10'] test_suites: all: venv/bin/pytest -n 2 -vvs @@ -27,7 +27,7 @@ jobs: parameters: job_name: macos1015_cpython image_name: macos-10.15 - python_versions: ['3.6', '3.7', '3.8', '3.9', '3.10'] + python_versions: ['3.8', '3.9', '3.10'] test_suites: all: venv/bin/pytest -n 2 -vvs @@ -35,7 +35,7 @@ jobs: parameters: job_name: macos11_cpython image_name: macos-11 - python_versions: ['3.7', '3.8', '3.9', '3.10'] + python_versions: ['3.8', '3.9', '3.10'] test_suites: all: venv/bin/pytest -n 2 -vvs @@ -43,7 +43,7 @@ jobs: parameters: job_name: win2019_cpython image_name: windows-2019 - python_versions: ['3.6', '3.7', '3.8', '3.9', '3.10'] + python_versions: ['3.8', '3.9', '3.10'] test_suites: all: venv\Scripts\pytest -n 2 -vvs @@ -51,6 +51,6 @@ jobs: parameters: job_name: win2022_cpython image_name: windows-2022 - python_versions: ['3.7', '3.8', '3.9', '3.10'] + python_versions: ['3.8', '3.9', '3.10'] test_suites: all: venv\Scripts\pytest -n 2 -vvs From 44d64fda495249869b5c4b2ee2e3c34461ddf3f1 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Tue, 14 Jun 2022 10:15:49 +0200 Subject: [PATCH 04/19] Improve README Signed-off-by: Philippe Ombredanne --- README.rst | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/README.rst b/README.rst index 8ea50de2..ce50be6a 100644 --- a/README.rst +++ b/README.rst @@ -24,6 +24,19 @@ The goal of python-inspector is to be a comprehensive library that can handle every style of Python package layouts, manifests and lockfiles. +Usage +-------- + +- Install with pip:: + + pip install python-inspector + +- Run a command line with:: + + dad --help + + + Its companion libraries are: - ``pip-requirements-parser``, a mostly correct pip requirements parsing From 9b3ade57c23d6d803bf30adccc73f6d5b75a190b Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Tue, 14 Jun 2022 10:16:01 +0200 Subject: [PATCH 05/19] Add missing import Signed-off-by: Philippe Ombredanne --- etc/scripts/utils_requirements.py | 1 + 1 file changed, 1 insertion(+) diff --git a/etc/scripts/utils_requirements.py b/etc/scripts/utils_requirements.py index 7c99a33b..db7e0ee2 100644 --- a/etc/scripts/utils_requirements.py +++ b/etc/scripts/utils_requirements.py @@ -8,6 +8,7 @@ # See https://github.com/nexB/skeleton for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # +import os import re import subprocess From c79554b2c3fc61c2c9941a81a925b1cb6542452e Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Tue, 14 Jun 2022 10:16:55 +0200 Subject: [PATCH 06/19] Add pinned requirements Signed-off-by: Philippe Ombredanne --- requirements-dev.txt | 41 +++++++++++++++++++++++++++++++++++++++++ requirements.txt | 27 +++++++++++++++++++++++++++ 2 files changed, 68 insertions(+) diff --git a/requirements-dev.txt b/requirements-dev.txt index e69de29b..6514e9e2 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -0,0 +1,41 @@ +aboutcode-toolkit==7.0.2 +black==22.3.0 +bleach==4.1.0 +boolean.py==4.0 +cffi==1.15.0 +cryptography==37.0.2 +dataclasses==0.8 +docutils==0.18.1 +et-xmlfile==1.1.0 +execnet==1.9.0 +importlib-resources==5.4.0 +iniconfig==1.1.1 +isort==5.10.1 +jeepney==0.7.1 +jinja2==3.0.3 +keyring==23.4.1 +license-expression==30.0.0 +markupsafe==2.0.1 +mypy-extensions==0.4.3 +openpyxl==3.0.10 +pathspec==0.9.0 +pkginfo==1.8.3 +platformdirs==2.4.0 +pluggy==1.0.0 +py==1.11.0 +pycodestyle==2.8.0 +pycparser==2.21 +pygments==2.12.0 +pytest==7.0.1 +pytest-forked==1.4.0 +pytest-xdist==2.5.0 +readme-renderer==34.0 +requests-toolbelt==0.9.1 +rfc3986==1.5.0 +secretstorage==3.3.2 +six==1.16.0 +tomli==1.2.3 +tqdm==4.64.0 +twine==3.8.0 +typed-ast==1.5.4 +webencodings==0.5.1 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index e69de29b..cb292414 100644 --- a/requirements.txt +++ b/requirements.txt @@ -0,0 +1,27 @@ +attrs==21.4.0 +beautifulsoup4==4.11.1 +certifi==2022.5.18.1 +charset-normalizer==2.0.12 +click==8.0.4 +colorama==0.4.4 +commoncode==30.2.0 +dparse2==0.6.1 +idna==3.3 +importlib-metadata==4.8.3 +intbitset==3.0.1 +packageurl-python==0.9.9 +packaging==21.3 +pip-requirements-parser==31.2.0 +pkginfo2==30.0.0 +pyparsing==3.0.9 +PyYAML==6.0 +requests==2.27.1 +resolvelib==0.8.1 +saneyaml==0.5.2 +soupsieve==2.3.2.post1 +text-unidecode==1.3 +toml==0.10.2 +typing==3.6.6 +typing_extensions==4.1.1 +urllib3==1.26.9 +zipp==3.6.0 From 7237aeef4865c0874f1bdb8d320ab3ad95eb3b9a Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Tue, 14 Jun 2022 11:11:11 +0200 Subject: [PATCH 07/19] Adjust code for style and format Signed-off-by: Philippe Ombredanne --- src/python_inspector/utils_pip_compatibility_tags.py | 3 ++- src/python_inspector/utils_pypi.py | 4 ++-- tests/test_resolution.py | 6 +++--- tests/test_utils_pip_compatibility_tags.py | 4 +++- 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/src/python_inspector/utils_pip_compatibility_tags.py b/src/python_inspector/utils_pip_compatibility_tags.py index 59b9b8b4..122ae59a 100644 --- a/src/python_inspector/utils_pip_compatibility_tags.py +++ b/src/python_inspector/utils_pip_compatibility_tags.py @@ -49,13 +49,14 @@ def _mac_platforms(arch): if match: name, major, minor, actual_arch = match.groups() mac_version = (int(major), int(minor)) + len_mac = len("macosx_") arches = [ # Since we have always only checked that the platform starts # with "macosx", for backwards-compatibility we extract the # actual prefix provided by the user in case they provided # something like "macosxcustom_". It may be good to remove # this as undocumented or deprecate it in the future. - "{}_{}".format(name, arch[len("macosx_") :]) + "{}_{}".format(name, arch[len_mac:]) for arch in mac_platforms(mac_version, actual_arch) ] else: diff --git a/src/python_inspector/utils_pypi.py b/src/python_inspector/utils_pypi.py index ec5a9795..661a3895 100644 --- a/src/python_inspector/utils_pypi.py +++ b/src/python_inspector/utils_pypi.py @@ -1024,7 +1024,7 @@ def is_pure(self): def is_pure_wheel(filename): try: return Wheel.from_filename(filename).is_pure() - except: + except Exception: return False @@ -1418,7 +1418,7 @@ def fetch_links(self, normalized_name): ) links = collect_urls(text) # TODO: keep sha256 - links = [l.partition("#sha256=") for l in links] + links = [lnk.partition("#sha256=") for lnk in links] links = [url for url, _, _sha256 in links] return links diff --git a/tests/test_resolution.py b/tests/test_resolution.py index 275921be..25ecd753 100644 --- a/tests/test_resolution.py +++ b/tests/test_resolution.py @@ -92,11 +92,11 @@ def test_resolvelib_with_tilde_requirement_using_json_api(): def test_pypi_simple_repo_in_repos(): - assert pypi_simple_repo_in_repos(repos=[PYPI_PUBLIC_REPO]) == True + assert pypi_simple_repo_in_repos(repos=[PYPI_PUBLIC_REPO]) def test_pypi_simple_repo_in_repos_not_present(): - assert pypi_simple_repo_in_repos(repos=[]) == False + assert not pypi_simple_repo_in_repos(repos=[]) def test_is_valid_version(): @@ -104,4 +104,4 @@ def test_is_valid_version(): requirements = {"flask": [Requirement("flask>2.0.0")]} bad_versions = [] identifier = "flask" - assert is_valid_version(parsed_version, requirements, identifier, bad_versions) == True + assert is_valid_version(parsed_version, requirements, identifier, bad_versions) diff --git a/tests/test_utils_pip_compatibility_tags.py b/tests/test_utils_pip_compatibility_tags.py index 5768403f..a492912c 100644 --- a/tests/test_utils_pip_compatibility_tags.py +++ b/tests/test_utils_pip_compatibility_tags.py @@ -95,7 +95,9 @@ def test_manylinux2010_implies_manylinux1(self, manylinux2010, manylinux1): Specifying manylinux2010 implies manylinux1. """ groups = {} - supported = utils_pip_compatibility_tags.get_supported(platforms=[manylinux2010]) + supported = utils_pip_compatibility_tags.get_supported( + platforms=[manylinux2010], + ) for tag in supported: groups.setdefault((tag.interpreter, tag.abi), []).append(tag.platform) From 45cb44651d0b347f2fda394e79b41d5dbcd7d3a3 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Tue, 14 Jun 2022 11:13:42 +0200 Subject: [PATCH 08/19] Adapt makefile to make proper checks Especially support line of up to 100 chars Signed-off-by: Philippe Ombredanne --- Makefile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 4a16890f..ddcabcec 100644 --- a/Makefile +++ b/Makefile @@ -18,7 +18,7 @@ dev: isort: @echo "-> Apply isort changes to ensure proper imports ordering" - ${VENV}/bin/isort --sl src tests + ${VENV}/bin/isort --sl -l 100 src tests black: @echo "-> Apply black code formatter" @@ -32,9 +32,10 @@ valid: isort black check: @echo "-> Run pycodestyle (PEP8) validation" - @${ACTIVATE} pycodestyle --max-line-length=100 --exclude=venv,lib,thirdparty,docs,migrations,settings.py . + @${ACTIVATE} pycodestyle --max-line-length=110 \ + --exclude=.eggs,etc/scripts,src/_packagedcode,venv,lib,thirdparty,docs . @echo "-> Run isort imports ordering validation" - @${ACTIVATE} isort --sl --check-only . + @${ACTIVATE} isort --sl --check-only -l 100 src tests @echo "-> Run black validation" @${ACTIVATE} black --check -l 100 From 3347103468e4280c095298e98c2ca9b7a617f6f1 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Tue, 14 Jun 2022 11:14:27 +0200 Subject: [PATCH 09/19] Add checks to CI Signed-off-by: Philippe Ombredanne --- azure-pipelines.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index e67d2e14..bf7c7022 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -13,7 +13,9 @@ jobs: image_name: ubuntu-18.04 python_versions: ['3.8', '3.9', '3.10'] test_suites: - all: venv/bin/pytest -n 2 -vvs + all: | + venv/bin/pytest -n 2 -vvs + make check - template: etc/ci/azure-posix.yml parameters: From db79dcf38adae644f9f4d7bf9c6d040e12362492 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Tue, 14 Jun 2022 17:09:54 +0200 Subject: [PATCH 10/19] Make format_resolution() args use as_tree only * Change format_resolution() args to return only either a tree or a parent/children structure. List is not needed. * rename resolution() to get_resolved_dependencies() *Streamline imports Signed-off-by: Philippe Ombredanne --- src/python_inspector/resolution.py | 115 ++++++++++------------- src/python_inspector/resolution.py.ABOUT | 2 +- 2 files changed, 52 insertions(+), 65 deletions(-) diff --git a/src/python_inspector/resolution.py b/src/python_inspector/resolution.py index ade40259..18dfc1ca 100644 --- a/src/python_inspector/resolution.py +++ b/src/python_inspector/resolution.py @@ -3,7 +3,7 @@ # ScanCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/scancode-toolkit for support or download. +# See https://github.com/nexB/python-inspector for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -12,9 +12,7 @@ import os from typing import List -import packaging.markers import packaging.requirements -import packaging.specifiers import packaging.utils import packaging.version import requests @@ -80,7 +78,7 @@ def get_preference( backtrack_causes, ): transitive = all(p is not None for _, p in information[identifier]) - return (transitive, identifier) + return transitive, identifier def get_versions_for_package(self, name, repo=None): """ @@ -147,7 +145,7 @@ def _iter_matches(self, identifier, requirements, incompatibilities): """ Return a list of candidates for the given identifier. """ - name, _, _ = identifier.partition("[") + name, _, _extras = identifier.partition("[") bad_versions = {c.version for c in incompatibilities[identifier]} extras = {e for r in requirements[identifier] for e in r.extras} if not self.repos: @@ -178,14 +176,17 @@ def _iter_dependencies(self, candidate): Yield dependencies for the given candidate. """ name = packaging.utils.canonicalize_name(candidate.name) + # TODO: handle extras https://github.com/nexB/python-inspector/issues/10 if candidate.extras: r = f"{name}=={candidate.version}" yield packaging.requirements.Requirement(r) + purl = PackageURL( type="pypi", name=name, version=str(candidate.version), ) + for r in self.get_requirements_for_package(purl, candidate): if r.marker is None: yield r @@ -208,7 +209,7 @@ def get_all_srcs(mapping, graph): def dfs(mapping, graph, src): """ - Return a recursive mapping of dependencies. + Return a nested mapping of dependencies. """ children = list(graph.iter_children(src)) src_purl = PackageURL( @@ -225,83 +226,69 @@ def dfs(mapping, graph, src): ) -def format_resolution(result): +def format_resolution(results, as_tree=False): """ - Return a formatted resolution. + Return a formatted resolution either as a tree or parent/children. """ - mapping = result.mapping - graph = result.graph - as_list = [ - str( - PackageURL( + mapping = results.mapping + graph = results.graph + + if not as_tree: + as_parent_children = [] + parents = mapping.keys() + for parent in parents: + parent_purl = PackageURL( type="pypi", - name=name, - version=str(candidate.version), + name=parent, + version=str(mapping[parent].version), ) - ) - for name, candidate in mapping.items() - ] - - as_parent_children = [] - parents = mapping.keys() - for parent in parents: - parent_purl = PackageURL( - type="pypi", - name=parent, - version=str(mapping[parent].version), - ) + dependencies = [] + for dependency in graph.iter_children(parent): + dep_purl = PackageURL( + type="pypi", + name=dependency, + version=str(mapping[dependency].version), + ) + dependencies.append(str(dep_purl)) + dependencies.sort() + parent_children = dict(package=str(parent_purl), dependencies=dependencies) + as_parent_children.append(parent_children) + as_parent_children.sort(key=lambda d: d["package"]) + return as_parent_children + else: dependencies = [] - for dependency in graph.iter_children(parent): - dep_purl = PackageURL( - type="pypi", - name=dependency, - version=str(mapping[dependency].version), - ) - dependencies.append(str(dep_purl)) - dependencies.sort() - parent_children = dict(package=str(parent_purl), dependencies=dependencies) - as_parent_children.append(parent_children) - - srcs = list(get_all_srcs(mapping=mapping, graph=graph)) - dependencies = [] - for src in srcs: - dependencies.append(dfs(mapping=mapping, graph=graph, src=src)) + for src in get_all_srcs(mapping=mapping, graph=graph): + dependencies.append(dfs(mapping=mapping, graph=graph, src=src)) - as_list.sort() - as_parent_children.sort(key=lambda d: d["package"]) - dependencies.sort(key=lambda d: d["package"]) - as_tree = dict(dependencies=dependencies) - return as_list, as_parent_children, as_tree + dependencies.sort(key=lambda d: d["package"]) + return dependencies def pypi_simple_repo_in_repos(repos: PypiSimpleRepository): """ Return True if simple pypi index_url is present in any of the repos """ - for repo in repos: - if repo.index_url == PYPI_SIMPLE_URL: - return True - return False + return any(repo.index_url == PYPI_SIMPLE_URL for repo in repos) -def resolution( +def get_resolved_dependencies( requirements: List[Requirement], environment: Environment = None, repos: List[PypiSimpleRepository] = [], - return_as_parent_children: bool = True, - return_as_tree: bool = False, - return_as_list: bool = False, + as_tree: bool = False, ): """ - Return a resolution for the given requirements. + Return resolved dependencies of a ``requirements`` list of Requirement for + an ``enviroment`` Environment. The resolved dependencies are formatted as + parent/children or a nested tree if ``as_tree`` is True """ if repos and not pypi_simple_repo_in_repos(repos): repos.append(PYPI_PUBLIC_REPO) - resolver = Resolver(PythonInputProvider(environment, repos), BaseReporter()) - as_list, as_parent_children, as_tree = format_resolution(resolver.resolve(requirements)) - if return_as_parent_children: - return as_parent_children - if return_as_tree: - return as_tree - if return_as_list: - return as_list + + resolver = Resolver( + provider=PythonInputProvider(environment, repos), + reporter=BaseReporter(), + ) + results = resolver.resolve(requirements=requirements) + results = format_resolution(results, as_tree=as_tree) + return results diff --git a/src/python_inspector/resolution.py.ABOUT b/src/python_inspector/resolution.py.ABOUT index 2b9f0090..6c23ef8b 100644 --- a/src/python_inspector/resolution.py.ABOUT +++ b/src/python_inspector/resolution.py.ABOUT @@ -9,6 +9,6 @@ subpath: tests/functional/python/test_resolvers_python.py download_url: https://github.com/sarugaku/resolvelib/blob/a5ae68140afac49dd1a1a8e87eff9550db4a586b/tests/functional/python/test_resolvers_python.py copyright: Copyright (c) 2018, Tzu-ping Chung license_expression: isc -notes: The PythonInputProvider is copied and heavily modified from +notes: The PythonInputProvider is copied in part and heavily modified from resolvelib tests/functional/python/test_resolvers_python.py \ No newline at end of file From 048d6b6195eaa1e94424d362ad3288b6ae0f6057 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Tue, 14 Jun 2022 17:12:58 +0200 Subject: [PATCH 11/19] * Enhance results header data structure * Change resuults returned by resolve() to be a tuple of (initial_requirements, resolved_dependencies) * Refine doc and doc strings * Use stderr for CLI messages Signed-off-by: Philippe Ombredanne --- src/python_inspector/resolve_cli.py | 138 +++++++++++++++++++--------- 1 file changed, 93 insertions(+), 45 deletions(-) diff --git a/src/python_inspector/resolve_cli.py b/src/python_inspector/resolve_cli.py index 610cee1e..cd2f3306 100644 --- a/src/python_inspector/resolve_cli.py +++ b/src/python_inspector/resolve_cli.py @@ -18,10 +18,15 @@ from python_inspector import dependencies from python_inspector import utils_pypi from python_inspector.cli_utils import FileOptionType -from python_inspector.resolution import resolution +from python_inspector.resolution import get_resolved_dependencies TRACE = False +__version__ = "0.5.0" + +DEFAULT_PYTHON_VERSION = "38" +PYPI_SIMPLE_URL = "https://pypi.org/simple" + @click.command() @click.option( @@ -43,7 +48,7 @@ metavar="SPECIFIER", multiple=True, required=False, - help="Package specifier such as django==1.2.3. " "This option can be used multiple times.", + help="Package specifier such as django==1.2.3. This option can be used multiple times.", ) @click.option( "-p", @@ -51,8 +56,7 @@ "python_version", type=click.Choice(utils_pypi.PYTHON_VERSIONS), metavar="PYVER", - # TODO: Make default the current Python version - default="38", # utils_pypi.PYTHON_VERSIONS, + default=DEFAULT_PYTHON_VERSION, show_default=True, help="Python version to use for dependency resolution.", ) @@ -64,7 +68,7 @@ metavar="OS", default="linux", show_default=True, - help="OS to use for dependency resolution. ", + help="OS to use for dependency resolution.", ) @click.option( "--index-url", @@ -72,8 +76,9 @@ type=str, metavar="INDEX", show_default=True, + default=tuple([PYPI_SIMPLE_URL]), multiple=True, - help="PyPI index URL(s) to use in order of preference. " + help="PyPI simple index URL(s) to use in order of preference. " "This option can be used multiple times.", ) @click.option( @@ -88,6 +93,7 @@ @click.option( "--use-cached-index", is_flag=True, + hidden=True, help="Use cached on-disk PyPI package indexes and do not refetch if present.", ) @click.option( @@ -104,7 +110,7 @@ def resolve_dependencies( operating_system, index_urls, json_output, - use_cached_index, + use_cached_index=False, debug=TRACE, ): """ @@ -112,30 +118,27 @@ def resolve_dependencies( and SPECIFIER(s) and save the results as JSON to FILE. Resolve the dependencies for the requested ``--python-version`` PYVER and - ``--operating_system`` OS combination defaulting to the current version and OS. + ``--operating_system`` OS combination defaulting Python version 3.8 and + linux OS. + + Download from the provided PyPI simple --index-url INDEX(s) URLs defaulting + to PyPI.org - Download from the provided PyPI simple --index-url INDEX(s) URLs. Error and progress are printed to stderr. - Default environment is the Python version - 3.8 and OS - linux. + 1) Without an --index-url is provided, this tool uses the PyPI JSON API. - 1) If no index_url is provided, the PyPI JSON API is used and environment will be ignored in that case. + For example, display the results of resolving the dependencies for flask==2.1.2 + on screen:: - For example: dad --spec "flask==2.1.2" --json - - 2) If an index_url is provided, the environment will be used to resolve the dependencies. - (If no environment is provided default environment will be used.) - - For example: - dad --spec "flask==2.1.2" --index-url https://pypi.org/simple --json - + 2) If an --index-url is provided, it is used to resolve the dependencies:: - For example:: - dad --spec "flask" --requirement etc/scripts/requirements.txt --json - + dad --spec "flask==2.1.2" --index-url https://pypi.org/simple --json - """ - # FIXME: Use stderr and click.secho - print(f"Resolving dependencies...") + click.secho(f"Resolving dependencies...") # TODO: deduplicate me direct_dependencies = [] @@ -149,13 +152,13 @@ def resolve_dependencies( direct_dependencies.append(dep) if not direct_dependencies: - print("Error: no requirements requested.") + click.secho("Error: no requirements requested.") sys.exit(1) if debug: - print("direct_dependencies:") + click.secho("direct_dependencies:") for dep in direct_dependencies: - print(" ", dep) + click.secho(" ", dep) # create a resolution environments environment = utils_pypi.Environment.from_pyver_and_os( @@ -163,7 +166,7 @@ def resolve_dependencies( ) if debug: - print("environment:", environment) + click.secho("environment:", environment) # Collect PyPI repos repos = [] @@ -181,42 +184,87 @@ def resolve_dependencies( repos.append(repo) if debug: - print("repos:") + click.secho("repos:") for repo in repos: - print(" ", repo) + click.secho(" ", repo) # resolve dependencies proper - resolved_dependencies = resolve(direct_dependencies, environment, repos) - write_output(results=resolved_dependencies, json_output=json_output) + requirements, resolved_dependencies = resolve( + direct_dependencies=direct_dependencies, + environment=environment, + repos=repos, + as_tree=False, + ) + + cli_options = [f"--requirement {rf}" for rf in requirement_files] + cli_options += [f"--specifier {sp}" for sp in specifiers] + cli_options += [f"--index-url {iu}" for iu in index_urls] + cli_options += [f"--python-version {python_version}"] + cli_options += [f"--operating-system {operating_system}"] + cli_options += ["--json "] + + notice = ( + "Dependency tree generated with python-inspector.\n" + "python-inspector is a free software tool from nexB Inc. and others.\n" + "Visit https://github.com/nexB/scancode-toolkit/ for support and download." + ) + + headers = dict( + tool_name="dad", + tool_homepageurl="https://github.com/nexB/python-inspector", + tool_version=__version__, + options=cli_options, + notice=notice, + warnings=[], + errors=[], + ) + + write_output( + headers=headers, + requirements=requirements, + resolved_dependencies=resolved_dependencies, + json_output=json_output, + ) if debug: - print("done!") + click.secho("done!") -def resolve(direct_dependencies, environment, repos): +def resolve(direct_dependencies, environment, repos, as_tree=False): """ Resolve dependencies given a ``direct_dependencies`` list of - DependentPackage and return SOMETHING TBD. + DependentPackage and return a tuple of (initial_requirements, + resolved_dependencies). """ - reqs = [Requirement(d.extracted_requirement) for d in direct_dependencies] - as_parent_children = resolution(reqs, environment, repos) - - return dict( - headers=[dict(tool="dad")], - requirements=[d.to_dict() for d in direct_dependencies], - resolved_dependencies=dict( - as_parent_children=as_parent_children, - ), + requirements = [ + Requirement(requirement_string=d.extracted_requirement) for d in direct_dependencies + ] + resolved_dependencies = get_resolved_dependencies( + requirements=requirements, + environment=environment, + repos=repos, + as_tree=as_tree, ) + initial_requirements = [d.to_dict() for d in direct_dependencies] -def write_output(results, json_output): + return initial_requirements, resolved_dependencies + + +def write_output(headers, requirements, resolved_dependencies, json_output): """ - Write headers, and resolved dependency results to ``output_file`` + Write headers, requirements and resolved_dependencies as JSON to ``json_output``. + Return the output data. """ - # TODO : create tree, add headers - json.dump(results, json_output, indent=2) + output = dict( + headers=headers, + requirements=requirements, + resolved_dependencies=resolved_dependencies, + ) + + json.dump(output, json_output, indent=2) + return output if __name__ == "__main__": From 16cdee912bf57fc685a8aa708793e96b3a00479d Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Tue, 14 Jun 2022 17:13:51 +0200 Subject: [PATCH 12/19] Recreate deps list for testing Signed-off-by: Philippe Ombredanne --- tests/test_resolution.py | 28 ++++++++++++---------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/tests/test_resolution.py b/tests/test_resolution.py index 25ecd753..38cf0098 100644 --- a/tests/test_resolution.py +++ b/tests/test_resolution.py @@ -12,26 +12,26 @@ import pytest from packaging.requirements import Requirement +from python_inspector.resolution import get_resolved_dependencies from python_inspector.resolution import is_valid_version from python_inspector.resolution import pypi_simple_repo_in_repos -from python_inspector.resolution import resolution from python_inspector.utils_pypi import PYPI_PUBLIC_REPO from python_inspector.utils_pypi import Environment @pytest.mark.online -def test_resolvelib_with_flask_and_python_310(): +def test_get_resolved_dependencies_with_flask_and_python_310(): req = [Requirement("flask==2.1.2")] - as_list = resolution( + results = get_resolved_dependencies( requirements=req, environment=Environment( python_version="310", operating_system="linux", ), repos=[PYPI_PUBLIC_REPO], - return_as_parent_children=False, - return_as_list=True, + as_tree=False, ) + as_list = [p["package"] for p in results] assert as_list == [ "pkg:pypi/click@8.1.3", "pkg:pypi/flask@2.1.2", @@ -45,18 +45,18 @@ def test_resolvelib_with_flask_and_python_310(): @pytest.mark.online -def test_resolvelib_with_flask_and_python_36(): +def test_get_resolved_dependencies_with_flask_and_python_36(): req = [Requirement("flask==2.1.2")] - as_list = resolution( + results = get_resolved_dependencies( requirements=req, environment=Environment( python_version="36", operating_system="linux", ), repos=[PYPI_PUBLIC_REPO], - return_as_parent_children=False, - return_as_list=True, + as_tree=False, ) + as_list = [p["package"] for p in results] assert as_list == [ "pkg:pypi/click@8.1.3", @@ -71,14 +71,10 @@ def test_resolvelib_with_flask_and_python_36(): @pytest.mark.online -def test_resolvelib_with_tilde_requirement_using_json_api(): +def test_get_resolved_dependencies_with_tilde_requirement_using_json_api(): req = [Requirement("flask~=2.1.2")] - as_list = resolution( - requirements=req, - return_as_parent_children=False, - return_as_list=True, - ) - + results = get_resolved_dependencies(requirements=req, as_tree=False) + as_list = [p["package"] for p in results] assert as_list == [ "pkg:pypi/click@8.1.3", "pkg:pypi/flask@2.1.2", From da7057f6901819600aec81c138ddb67cf07cb0e0 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Tue, 14 Jun 2022 17:14:59 +0200 Subject: [PATCH 13/19] Refine CLI testing * Make it data driven and test the whole result againsta JSON file * Add new test for a requirements files * Use regeneration of fixtures with the PYINSP_REGEN_TEST_FIXTURES=yes environment variable. Signed-off-by: Philippe Ombredanne --- tests/data/default-url-expected.json | 34 ++ tests/data/pinned-requirements.txt | 27 ++ .../pinned-requirements.txt-expected.json | 399 ++++++++++++++++++ tests/data/single-url-expected.json | 34 ++ tests/data/tilde_req-expected.json | 35 ++ tests/test_cli.py | 160 +++++-- 6 files changed, 663 insertions(+), 26 deletions(-) create mode 100644 tests/data/default-url-expected.json create mode 100644 tests/data/pinned-requirements.txt create mode 100644 tests/data/pinned-requirements.txt-expected.json create mode 100644 tests/data/single-url-expected.json create mode 100644 tests/data/tilde_req-expected.json diff --git a/tests/data/default-url-expected.json b/tests/data/default-url-expected.json new file mode 100644 index 00000000..aff991a1 --- /dev/null +++ b/tests/data/default-url-expected.json @@ -0,0 +1,34 @@ +{ + "headers": { + "tool_name": "dad", + "tool_homepageurl": "https://github.com/nexB/python-inspector", + "tool_version": "0.5.0", + "options": [ + "--specifier zipp==3.8.0", + "--index-url https://pypi.org/simple", + "--python-version 38", + "--operating-system linux", + "--json " + ], + "notice": "Dependency tree generated with python-inspector.\npython-inspector is a free software tool from nexB Inc. and others.\nVisit https://github.com/nexB/scancode-toolkit/ for support and download.", + "warnings": [], + "errors": [] + }, + "requirements": [ + { + "purl": "pkg:pypi/zipp@3.8.0", + "extracted_requirement": "zipp==3.8.0", + "scope": "install", + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "resolved_package": {} + } + ], + "resolved_dependencies": [ + { + "package": "pkg:pypi/zipp@3.8.0", + "dependencies": [] + } + ] +} \ No newline at end of file diff --git a/tests/data/pinned-requirements.txt b/tests/data/pinned-requirements.txt new file mode 100644 index 00000000..cb292414 --- /dev/null +++ b/tests/data/pinned-requirements.txt @@ -0,0 +1,27 @@ +attrs==21.4.0 +beautifulsoup4==4.11.1 +certifi==2022.5.18.1 +charset-normalizer==2.0.12 +click==8.0.4 +colorama==0.4.4 +commoncode==30.2.0 +dparse2==0.6.1 +idna==3.3 +importlib-metadata==4.8.3 +intbitset==3.0.1 +packageurl-python==0.9.9 +packaging==21.3 +pip-requirements-parser==31.2.0 +pkginfo2==30.0.0 +pyparsing==3.0.9 +PyYAML==6.0 +requests==2.27.1 +resolvelib==0.8.1 +saneyaml==0.5.2 +soupsieve==2.3.2.post1 +text-unidecode==1.3 +toml==0.10.2 +typing==3.6.6 +typing_extensions==4.1.1 +urllib3==1.26.9 +zipp==3.6.0 diff --git a/tests/data/pinned-requirements.txt-expected.json b/tests/data/pinned-requirements.txt-expected.json new file mode 100644 index 00000000..aef527dd --- /dev/null +++ b/tests/data/pinned-requirements.txt-expected.json @@ -0,0 +1,399 @@ +{ + "headers": { + "tool_name": "dad", + "tool_homepageurl": "https://github.com/nexB/python-inspector", + "tool_version": "0.5.0", + "options": [ + "--requirement /home/pombreda/w421/python-inspector/tests/data/pinned-requirements.txt", + "--index-url https://pypi.org/simple", + "--python-version 38", + "--operating-system linux", + "--json " + ], + "notice": "Dependency tree generated with python-inspector.\npython-inspector is a free software tool from nexB Inc. and others.\nVisit https://github.com/nexB/scancode-toolkit/ for support and download.", + "warnings": [], + "errors": [] + }, + "requirements": [ + { + "purl": "pkg:pypi/attrs@21.4.0", + "extracted_requirement": "attrs==21.4.0", + "scope": "install", + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "resolved_package": {} + }, + { + "purl": "pkg:pypi/beautifulsoup4@4.11.1", + "extracted_requirement": "beautifulsoup4==4.11.1", + "scope": "install", + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "resolved_package": {} + }, + { + "purl": "pkg:pypi/certifi@2022.5.18.1", + "extracted_requirement": "certifi==2022.5.18.1", + "scope": "install", + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "resolved_package": {} + }, + { + "purl": "pkg:pypi/charset-normalizer@2.0.12", + "extracted_requirement": "charset-normalizer==2.0.12", + "scope": "install", + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "resolved_package": {} + }, + { + "purl": "pkg:pypi/click@8.0.4", + "extracted_requirement": "click==8.0.4", + "scope": "install", + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "resolved_package": {} + }, + { + "purl": "pkg:pypi/colorama@0.4.4", + "extracted_requirement": "colorama==0.4.4", + "scope": "install", + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "resolved_package": {} + }, + { + "purl": "pkg:pypi/commoncode@30.2.0", + "extracted_requirement": "commoncode==30.2.0", + "scope": "install", + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "resolved_package": {} + }, + { + "purl": "pkg:pypi/dparse2@0.6.1", + "extracted_requirement": "dparse2==0.6.1", + "scope": "install", + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "resolved_package": {} + }, + { + "purl": "pkg:pypi/idna@3.3", + "extracted_requirement": "idna==3.3", + "scope": "install", + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "resolved_package": {} + }, + { + "purl": "pkg:pypi/importlib-metadata@4.8.3", + "extracted_requirement": "importlib-metadata==4.8.3", + "scope": "install", + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "resolved_package": {} + }, + { + "purl": "pkg:pypi/intbitset@3.0.1", + "extracted_requirement": "intbitset==3.0.1", + "scope": "install", + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "resolved_package": {} + }, + { + "purl": "pkg:pypi/packageurl-python@0.9.9", + "extracted_requirement": "packageurl-python==0.9.9", + "scope": "install", + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "resolved_package": {} + }, + { + "purl": "pkg:pypi/packaging@21.3", + "extracted_requirement": "packaging==21.3", + "scope": "install", + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "resolved_package": {} + }, + { + "purl": "pkg:pypi/pip-requirements-parser@31.2.0", + "extracted_requirement": "pip-requirements-parser==31.2.0", + "scope": "install", + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "resolved_package": {} + }, + { + "purl": "pkg:pypi/pkginfo2@30.0.0", + "extracted_requirement": "pkginfo2==30.0.0", + "scope": "install", + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "resolved_package": {} + }, + { + "purl": "pkg:pypi/pyparsing@3.0.9", + "extracted_requirement": "pyparsing==3.0.9", + "scope": "install", + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "resolved_package": {} + }, + { + "purl": "pkg:pypi/pyyaml@6.0", + "extracted_requirement": "PyYAML==6.0", + "scope": "install", + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "resolved_package": {} + }, + { + "purl": "pkg:pypi/requests@2.27.1", + "extracted_requirement": "requests==2.27.1", + "scope": "install", + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "resolved_package": {} + }, + { + "purl": "pkg:pypi/resolvelib@0.8.1", + "extracted_requirement": "resolvelib==0.8.1", + "scope": "install", + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "resolved_package": {} + }, + { + "purl": "pkg:pypi/saneyaml@0.5.2", + "extracted_requirement": "saneyaml==0.5.2", + "scope": "install", + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "resolved_package": {} + }, + { + "purl": "pkg:pypi/soupsieve@2.3.2.post1", + "extracted_requirement": "soupsieve==2.3.2.post1", + "scope": "install", + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "resolved_package": {} + }, + { + "purl": "pkg:pypi/text-unidecode@1.3", + "extracted_requirement": "text-unidecode==1.3", + "scope": "install", + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "resolved_package": {} + }, + { + "purl": "pkg:pypi/toml@0.10.2", + "extracted_requirement": "toml==0.10.2", + "scope": "install", + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "resolved_package": {} + }, + { + "purl": "pkg:pypi/typing@3.6.6", + "extracted_requirement": "typing==3.6.6", + "scope": "install", + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "resolved_package": {} + }, + { + "purl": "pkg:pypi/typing-extensions@4.1.1", + "extracted_requirement": "typing_extensions==4.1.1", + "scope": "install", + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "resolved_package": {} + }, + { + "purl": "pkg:pypi/urllib3@1.26.9", + "extracted_requirement": "urllib3==1.26.9", + "scope": "install", + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "resolved_package": {} + }, + { + "purl": "pkg:pypi/zipp@3.6.0", + "extracted_requirement": "zipp==3.6.0", + "scope": "install", + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "resolved_package": {} + } + ], + "resolved_dependencies": [ + { + "package": "pkg:pypi/attrs@21.4.0", + "dependencies": [] + }, + { + "package": "pkg:pypi/beautifulsoup4@4.11.1", + "dependencies": [ + "pkg:pypi/soupsieve@2.3.2.post1" + ] + }, + { + "package": "pkg:pypi/certifi@2022.5.18.1", + "dependencies": [] + }, + { + "package": "pkg:pypi/charset-normalizer@2.0.12", + "dependencies": [] + }, + { + "package": "pkg:pypi/click@8.0.4", + "dependencies": [] + }, + { + "package": "pkg:pypi/colorama@0.4.4", + "dependencies": [] + }, + { + "package": "pkg:pypi/commoncode@30.2.0", + "dependencies": [ + "pkg:pypi/attrs@21.4.0", + "pkg:pypi/beautifulsoup4@4.11.1", + "pkg:pypi/click@8.0.4", + "pkg:pypi/intbitset@3.0.1", + "pkg:pypi/requests@2.27.1", + "pkg:pypi/saneyaml@0.5.2", + "pkg:pypi/text-unidecode@1.3" + ] + }, + { + "package": "pkg:pypi/dparse2@0.6.1", + "dependencies": [ + "pkg:pypi/packaging@21.3", + "pkg:pypi/pyyaml@6.0", + "pkg:pypi/toml@0.10.2" + ] + }, + { + "package": "pkg:pypi/idna@3.3", + "dependencies": [] + }, + { + "package": "pkg:pypi/importlib-metadata@4.8.3", + "dependencies": [ + "pkg:pypi/zipp@3.6.0" + ] + }, + { + "package": "pkg:pypi/intbitset@3.0.1", + "dependencies": [] + }, + { + "package": "pkg:pypi/packageurl-python@0.9.9", + "dependencies": [] + }, + { + "package": "pkg:pypi/packaging@21.3", + "dependencies": [ + "pkg:pypi/pyparsing@3.0.9" + ] + }, + { + "package": "pkg:pypi/pip-requirements-parser@31.2.0", + "dependencies": [ + "pkg:pypi/packaging@21.3" + ] + }, + { + "package": "pkg:pypi/pkginfo2@30.0.0", + "dependencies": [] + }, + { + "package": "pkg:pypi/pyparsing@3.0.9", + "dependencies": [] + }, + { + "package": "pkg:pypi/pyyaml@6.0", + "dependencies": [] + }, + { + "package": "pkg:pypi/requests@2.27.1", + "dependencies": [ + "pkg:pypi/certifi@2022.5.18.1", + "pkg:pypi/charset-normalizer@2.0.12", + "pkg:pypi/idna@3.3", + "pkg:pypi/urllib3@1.26.9" + ] + }, + { + "package": "pkg:pypi/resolvelib@0.8.1", + "dependencies": [] + }, + { + "package": "pkg:pypi/saneyaml@0.5.2", + "dependencies": [ + "pkg:pypi/pyyaml@6.0" + ] + }, + { + "package": "pkg:pypi/soupsieve@2.3.2.post1", + "dependencies": [] + }, + { + "package": "pkg:pypi/text-unidecode@1.3", + "dependencies": [] + }, + { + "package": "pkg:pypi/toml@0.10.2", + "dependencies": [] + }, + { + "package": "pkg:pypi/typing-extensions@4.1.1", + "dependencies": [] + }, + { + "package": "pkg:pypi/typing@3.6.6", + "dependencies": [] + }, + { + "package": "pkg:pypi/urllib3@1.26.9", + "dependencies": [] + }, + { + "package": "pkg:pypi/zipp@3.6.0", + "dependencies": [] + } + ] +} \ No newline at end of file diff --git a/tests/data/single-url-expected.json b/tests/data/single-url-expected.json new file mode 100644 index 00000000..aff991a1 --- /dev/null +++ b/tests/data/single-url-expected.json @@ -0,0 +1,34 @@ +{ + "headers": { + "tool_name": "dad", + "tool_homepageurl": "https://github.com/nexB/python-inspector", + "tool_version": "0.5.0", + "options": [ + "--specifier zipp==3.8.0", + "--index-url https://pypi.org/simple", + "--python-version 38", + "--operating-system linux", + "--json " + ], + "notice": "Dependency tree generated with python-inspector.\npython-inspector is a free software tool from nexB Inc. and others.\nVisit https://github.com/nexB/scancode-toolkit/ for support and download.", + "warnings": [], + "errors": [] + }, + "requirements": [ + { + "purl": "pkg:pypi/zipp@3.8.0", + "extracted_requirement": "zipp==3.8.0", + "scope": "install", + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "resolved_package": {} + } + ], + "resolved_dependencies": [ + { + "package": "pkg:pypi/zipp@3.8.0", + "dependencies": [] + } + ] +} \ No newline at end of file diff --git a/tests/data/tilde_req-expected.json b/tests/data/tilde_req-expected.json new file mode 100644 index 00000000..7809e80f --- /dev/null +++ b/tests/data/tilde_req-expected.json @@ -0,0 +1,35 @@ +{ + "headers": { + "tool_name": "dad", + "tool_homepageurl": "https://github.com/nexB/python-inspector", + "tool_version": "0.5.0", + "options": [ + "--specifier zipp~=3.8.0", + "--index-url https://pypi.org/simple", + "--index-url https://thirdparty.aboutcode.org/pypi/simple/", + "--python-version 38", + "--operating-system linux", + "--json " + ], + "notice": "Dependency tree generated with python-inspector.\npython-inspector is a free software tool from nexB Inc. and others.\nVisit https://github.com/nexB/scancode-toolkit/ for support and download.", + "warnings": [], + "errors": [] + }, + "requirements": [ + { + "purl": "pkg:pypi/zipp", + "extracted_requirement": "zipp~=3.8.0", + "scope": "install", + "is_runtime": true, + "is_optional": false, + "is_resolved": false, + "resolved_package": {} + } + ], + "resolved_dependencies": [ + { + "package": "pkg:pypi/zipp@3.8.0", + "dependencies": [] + } + ] +} \ No newline at end of file diff --git a/tests/test_cli.py b/tests/test_cli.py index 13f9f321..3c4d52d9 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -9,46 +9,154 @@ # See https://aboutcode.org for more information about nexB OSS projects. # +import json +import os + import pytest from click.testing import CliRunner +from commoncode.testcase import FileDrivenTesting + +from python_inspector.resolve_cli import resolve_dependencies -from python_inspector.resolve_cli import resolve_dependencies as cli +# Used for tests to regenerate fixtures with regen=True +REGEN_TEST_FIXTURES = os.getenv("PYINSP_REGEN_TEST_FIXTURES", False) + +test_env = FileDrivenTesting() +test_env.test_data_dir = os.path.join(os.path.dirname(__file__), "data") @pytest.mark.online -def test_cli_with_json_api(): - runner = CliRunner() - result = runner.invoke( - cli, - ["--spec", "zipp==3.8.0", "--json", "-"], +def test_cli_with_default_urls(): + expected_file = test_env.get_test_loc("default-url-expected.json", must_exist=False) + specifier = "zipp==3.8.0" + check_specs_resolution( + specifier=specifier, + expected_file=expected_file, + regen=REGEN_TEST_FIXTURES, ) - assert result.exit_code == 0 @pytest.mark.online def test_cli_with_single_index_url(): - runner = CliRunner() - result = runner.invoke( - cli, - ["--spec", "zipp==3.8.0", "--index-url", "https://pypi.org/simple", "--json", "-"], + expected_file = test_env.get_test_loc("single-url-expected.json", must_exist=False) + specifier = "zipp==3.8.0" + extra_options = [ + "--index-url", + "https://pypi.org/simple", + ] + check_specs_resolution( + specifier=specifier, + expected_file=expected_file, + extra_options=extra_options, + regen=REGEN_TEST_FIXTURES, ) - assert result.exit_code == 0 @pytest.mark.online def test_cli_with_multiple_index_url_and_tilde_req(): - runner = CliRunner() - result = runner.invoke( - cli, - [ - "--spec", - "zipp~=3.8.0", - "--index-url", - "https://pypi.org/simple", - "--index-url", - "https://thirdparty.aboutcode.org/pypi/simple/", - "--json", - "-", - ], + expected_file = test_env.get_test_loc("tilde_req-expected.json", must_exist=False) + specifier = "zipp~=3.8.0" + extra_options = [ + "--index-url", + "https://pypi.org/simple", + "--index-url", + "https://thirdparty.aboutcode.org/pypi/simple/", + ] + check_specs_resolution( + specifier=specifier, + expected_file=expected_file, + extra_options=extra_options, + regen=REGEN_TEST_FIXTURES, + ) + + +@pytest.mark.online +def test_cli_with_pinned_requirements_file(): + requirements_file = test_env.get_test_loc("pinned-requirements.txt") + expected_file = test_env.get_test_loc("pinned-requirements.txt-expected.json", must_exist=False) + check_requirements_resolution( + requirements_file=requirements_file, + expected_file=expected_file, + regen=REGEN_TEST_FIXTURES, ) - assert result.exit_code == 0 + + +def check_specs_resolution( + specifier, + expected_file, + extra_options=tuple(), + regen=REGEN_TEST_FIXTURES, +): + result_file = test_env.get_temp_file("json") + options = ["--specifier", specifier, "--json", result_file] + options.extend(extra_options) + run_cli(options=options) + check_json_results( + result_file=result_file, + expected_file=expected_file, + regen=regen, + ) + + +def check_requirements_resolution( + requirements_file, + expected_file, + extra_options=tuple(), + regen=REGEN_TEST_FIXTURES, +): + result_file = test_env.get_temp_file("json") + options = ["--requirement", requirements_file, "--json", result_file] + options.extend(extra_options) + run_cli(options=options) + check_json_results( + result_file=result_file, + expected_file=expected_file, + regen=regen, + ) + + +def check_json_results(result_file, expected_file, regen=REGEN_TEST_FIXTURES): + """ + Check the ``result_file`` JSON results against the ``expected_file`` + expected JSON results. + + If ``regen`` is True the expected_file WILL BE overwritten with the new + results from ``results_file``. This is convenient for updating tests + expectations. + """ + with open(result_file) as res: + results = json.load(res) + + if regen: + with open(expected_file, "w") as reg: + json.dump(results, reg, indent=2, separators=(",", ": ")) + expected = results + else: + with open(expected_file) as res: + expected = json.load(res) + + assert results == expected + + +def run_cli(options, cli=resolve_dependencies, expected_rc=0, env=None): + """ + Run a command line resolution. Return a click.testing.Result object. + """ + + if not env: + env = dict(os.environ) + + runner = CliRunner() + result = runner.invoke(cli, options, catch_exceptions=False, env=env) + + if result.exit_code != expected_rc: + output = result.output + error = f""" +Failure to run: +rc: {result.exit_code} +python-inspector {options} +output: +{output} +""" + assert result.exit_code == expected_rc, error + return result From 49df2cbca82f89b503287abbcb5b36a0af2e815a Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Tue, 14 Jun 2022 17:46:09 +0200 Subject: [PATCH 14/19] Plan for verbose output Signed-off-by: Philippe Ombredanne --- src/python_inspector/utils_pypi.py | 146 ++++++++++++++++++++++++----- 1 file changed, 124 insertions(+), 22 deletions(-) diff --git a/src/python_inspector/utils_pypi.py b/src/python_inspector/utils_pypi.py index 661a3895..484d5cec 100644 --- a/src/python_inspector/utils_pypi.py +++ b/src/python_inspector/utils_pypi.py @@ -164,7 +164,6 @@ def get_python_dot_version(version): CACHE_THIRDPARTY_DIR = ".cache/thirdparty" - ################################################################################ PYPI_SIMPLE_URL = "https://pypi.org/simple" @@ -190,7 +189,15 @@ class DistributionNotFound(Exception): pass -def download_wheel(name, version, environment, dest_dir=CACHE_THIRDPARTY_DIR, repos=tuple()): +def download_wheel( + name, + version, + environment, + dest_dir=CACHE_THIRDPARTY_DIR, + repos=tuple(), + verbose=False, + echo_func=None, +): """ Download the wheels binary distribution(s) of package ``name`` and ``version`` matching the ``environment`` Environment constraints into the @@ -226,7 +233,11 @@ def download_wheel(name, version, environment, dest_dir=CACHE_THIRDPARTY_DIR, re print( f" download_wheel: Getting wheel from index (or cache): {wheel.download_url}" ) - fetched_wheel_filename = wheel.download(dest_dir=dest_dir) + fetched_wheel_filename = wheel.download( + dest_dir=dest_dir, + verbose=verbose, + echo_func=echo_func, + ) fetched_wheel_filenames.append(fetched_wheel_filename) if fetched_wheel_filenames: @@ -236,7 +247,14 @@ def download_wheel(name, version, environment, dest_dir=CACHE_THIRDPARTY_DIR, re return fetched_wheel_filenames -def download_sdist(name, version, dest_dir=CACHE_THIRDPARTY_DIR, repos=tuple()): +def download_sdist( + name, + version, + dest_dir=CACHE_THIRDPARTY_DIR, + repos=tuple(), + verbose=False, + echo_func=None, +): """ Download the sdist source distribution of package ``name`` and ``version`` into the ``dest_dir`` directory. Return a fetched filename or None. @@ -267,7 +285,11 @@ def download_sdist(name, version, dest_dir=CACHE_THIRDPARTY_DIR, repos=tuple()): if TRACE_DEEP: print(f" download_sdist: Getting sdist from index (or cache): {sdist.download_url}") - fetched_sdist_filename = package.sdist.download(dest_dir=dest_dir) + fetched_sdist_filename = package.sdist.download( + dest_dir=dest_dir, + verbose=verbose, + echo_func=echo_func, + ) if fetched_sdist_filename: # do not futher fetch from other repos if we find in first, typically PyPI @@ -522,7 +544,12 @@ def get_best_download_url(self, repos=tuple()): f" get_best_download_url: {self.filename} not found in {repo.index_url}" ) - def download(self, dest_dir=CACHE_THIRDPARTY_DIR): + def download( + self, + dest_dir=CACHE_THIRDPARTY_DIR, + verbose=False, + echo_func=None, + ): """ Download this distribution into `dest_dir` directory. Return the fetched filename. @@ -540,6 +567,8 @@ def download(self, dest_dir=CACHE_THIRDPARTY_DIR): dest_dir=dest_dir, filename=self.filename, as_text=False, + verbose=verbose, + echo_func=echo_func, ) return self.filename @@ -1358,7 +1387,12 @@ class PypiSimpleRepository: repr=False, ) - def _get_package_versions_map(self, name): + def _get_package_versions_map( + self, + name, + verbose=False, + echo_func=None, + ): """ Return a mapping of all available PypiPackage version for this package name. The mapping may be empty. It is ordered by version from oldest to newest @@ -1369,8 +1403,12 @@ def _get_package_versions_map(self, name): if not versions and normalized_name not in self.fetched_package_normalized_names: self.fetched_package_normalized_names.add(normalized_name) try: - links = self.fetch_links(normalized_name=normalized_name) - # note that thsi is sorted so the mapping is also sorted + links = self.fetch_links( + normalized_name=normalized_name, + verbose=verbose, + echo_func=echo_func, + ) + # note that this is sorted so the mapping is also sorted versions = { package.version: package for package in PypiPackage.packages_from_many_paths_or_urls(paths_or_urls=links) @@ -1385,27 +1423,59 @@ def _get_package_versions_map(self, name): return versions - def get_package_versions(self, name): + def get_package_versions( + self, + name, + verbose=False, + echo_func=None, + ): """ Return a mapping of all available PypiPackage version as{version: package} for this package name. The mapping may be empty but not None. It is sorted by version from oldest to newest. """ - return dict(self._get_package_versions_map(name)) + return dict( + self._get_package_versions_map( + name=name, + verbose=verbose, + echo_func=echo_func, + ) + ) - def get_package_version(self, name, version=None): + def get_package_version( + self, + name, + version=None, + verbose=False, + echo_func=None, + ): """ Return the PypiPackage with name and version or None. Return the latest PypiPackage version if version is None. """ if not version: - versions = list(self._get_package_versions_map(name).values()) + versions = list( + self._get_package_versions_map( + name=name, + verbose=verbose, + echo_func=echo_func, + ).values() + ) # return the latest version return versions and versions[-1] else: - return self._get_package_versions_map(name).get(version) - - def fetch_links(self, normalized_name): + return self._get_package_versions_map( + name=name, + verbose=verbose, + echo_func=echo_func, + ).get(version) + + def fetch_links( + self, + normalized_name, + verbose=False, + echo_func=None, + ): """ Return a list of download link URLs found in a PyPI simple index for package name using the `index_url` of this repository. @@ -1415,6 +1485,8 @@ def fetch_links(self, normalized_name): path_or_url=package_url, as_text=True, force=not self.use_cached_index, + verbose=verbose, + echo_func=echo_func, ) links = collect_urls(text) # TODO: keep sha256 @@ -1427,7 +1499,6 @@ def fetch_links(self, normalized_name): DEFAULT_PYPI_REPOS = (PYPI_PUBLIC_REPO,) DEFAULT_PYPI_REPOS_BY_URL = {r.index_url: r for r in DEFAULT_PYPI_REPOS} - ################################################################################ # # Basic file and URL-based operations using a persistent file-based Cache @@ -1447,7 +1518,14 @@ class Cache: def __attrs_post_init__(self): os.makedirs(self.directory, exist_ok=True) - def get(self, path_or_url, as_text=True, force=False): + def get( + self, + path_or_url, + as_text=True, + force=False, + verbose=False, + echo_func=None, + ): """ Return the content fetched from a ``path_or_url`` through the cache. Raise an Exception on errors. Treats the content as text if as_text is @@ -1460,7 +1538,12 @@ def get(self, path_or_url, as_text=True, force=False): if force or not os.path.exists(cached): if TRACE_DEEP: print(f" FILE CACHE MISS: {path_or_url}") - content = get_file_content(path_or_url=path_or_url, as_text=as_text) + content = get_file_content( + path_or_url=path_or_url, + as_text=as_text, + verbose=verbose, + echo_func=echo_func, + ) wmode = "w" if as_text else "wb" with open(cached, wmode) as fo: fo.write(content) @@ -1474,7 +1557,12 @@ def get(self, path_or_url, as_text=True, force=False): CACHE = Cache() -def get_file_content(path_or_url, as_text=True): +def get_file_content( + path_or_url, + as_text=True, + verbose=False, + echo_func=None, +): """ Fetch and return the content at `path_or_url` from either a local path or a remote URL. Return the content as bytes is `as_text` is False. @@ -1482,7 +1570,12 @@ def get_file_content(path_or_url, as_text=True): if path_or_url.startswith("https://"): if TRACE_DEEP: print(f"Fetching: {path_or_url}") - _headers, content = get_remote_file_content(url=path_or_url, as_text=as_text) + _headers, content = get_remote_file_content( + url=path_or_url, + as_text=as_text, + verbose=verbose, + echo_func=echo_func, + ) return content elif path_or_url.startswith("file://") or ( @@ -1517,6 +1610,8 @@ def get_remote_file_content( headers_only=False, headers=None, _delay=0, + verbose=False, + echo_func=None, ): """ Fetch and return a tuple of (headers, content) at `url`. Return content as a @@ -1532,7 +1627,10 @@ def get_remote_file_content( # using a GET with stream=True ensure we get the the final header from # several redirects and that we can ignore content there. A HEAD request may # not get us this last header - print(f" DOWNLOADING: {url}") + if verbose and not echo_func: + echo_func = print + if verbose: + echo_func(f"DOWNLOADING: {url}") with requests.get(url, allow_redirects=True, stream=True, headers=headers) as response: status = response.status_code if status != requests.codes.ok: # NOQA @@ -1561,6 +1659,8 @@ def fetch_and_save( dest_dir, filename, as_text=True, + verbose=False, + echo_func=None, ): """ Fetch content at ``path_or_url`` URL or path and save this to @@ -1571,6 +1671,8 @@ def fetch_and_save( content = CACHE.get( path_or_url=path_or_url, as_text=as_text, + verbose=verbose, + echo_func=echo_func, ) output = os.path.join(dest_dir, filename) wmode = "w" if as_text else "wb" From ebb8cc92d4445b26d946643bbf6d3544a1f6a1d5 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Tue, 14 Jun 2022 17:47:00 +0200 Subject: [PATCH 15/19] Use namespaced impors * This help better understand what comes from where. * Also add missing ISC license Signed-off-by: Philippe Ombredanne --- src/python_inspector/resolution.py | 41 ++++++++++++---------- src/python_inspector/resolution.py.LICENSE | 13 +++++++ 2 files changed, 35 insertions(+), 19 deletions(-) create mode 100644 src/python_inspector/resolution.py.LICENSE diff --git a/src/python_inspector/resolution.py b/src/python_inspector/resolution.py index 18dfc1ca..cfdaeb4e 100644 --- a/src/python_inspector/resolution.py +++ b/src/python_inspector/resolution.py @@ -1,8 +1,8 @@ # -# Copyright (c) nexB Inc. and others. All rights reserved. -# ScanCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# Copyright (c) Tzu-ping Chung , nexB Inc. and others. +# SPDX-License-Identifier: ISC AND Apache-2.0 +# derived and heavily modified from https://github.com/sarugaku/resolvelib + # See https://github.com/nexB/python-inspector for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -23,12 +23,7 @@ from resolvelib.reporters import BaseReporter from _packagedcode.pypi import PypiWheelHandler -from python_inspector.utils_pypi import CACHE_THIRDPARTY_DIR -from python_inspector.utils_pypi import PYPI_PUBLIC_REPO -from python_inspector.utils_pypi import PYPI_SIMPLE_URL -from python_inspector.utils_pypi import Environment -from python_inspector.utils_pypi import PypiSimpleRepository -from python_inspector.utils_pypi import download_wheel +from python_inspector import utils_pypi Candidate = collections.namedtuple("Candidate", "name version extras") @@ -63,6 +58,7 @@ def __init__(self, environment=None, repos=[]): self.dependencies_by_purl = {} def identify(self, requirement_or_candidate): + """Given a requirement, return an identifier for it. Overridden.""" name = packaging.utils.canonicalize_name(requirement_or_candidate.name) if requirement_or_candidate.extras: extras_str = ",".join(sorted(requirement_or_candidate.extras)) @@ -77,6 +73,8 @@ def get_preference( information, backtrack_causes, ): + """Produce a sort key for given requirement based on preference. Overridden.""" + transitive = all(p is not None for _, p in information[identifier]) return transitive, identifier @@ -106,14 +104,16 @@ def get_requirements_for_package(self, purl, candidate): Yield requirements for a package. """ if self.repos and self.environment: - wheels = download_wheel( + wheels = utils_pypi.download_wheel( name=candidate.name, version=str(candidate.version), environment=self.environment, repos=self.repos, ) for wheel in wheels: - deps = list(PypiWheelHandler.parse(os.path.join(CACHE_THIRDPARTY_DIR, wheel))) + deps = list( + PypiWheelHandler.parse(os.path.join(utils_pypi.CACHE_THIRDPARTY_DIR, wheel)) + ) assert len(deps) == 1 deps = deps[0].dependencies for dep in deps: @@ -133,7 +133,7 @@ def get_requirements_for_package(self, purl, candidate): def get_candidates(self, all_versions, requirements, identifier, bad_versions, name, extras): """ - Generate candidates for the given identifier. + Generate candidates for the given identifier. Overridden. """ for version in all_versions: parsed_version = packaging.version.parse(version) @@ -143,7 +143,7 @@ def get_candidates(self, all_versions, requirements, identifier, bad_versions, n def _iter_matches(self, identifier, requirements, incompatibilities): """ - Return a list of candidates for the given identifier. + Yield candidates for the given identifier, requirements and incompatibilities """ name, _, _extras = identifier.partition("[") bad_versions = {c.version for c in incompatibilities[identifier]} @@ -161,6 +161,7 @@ def _iter_matches(self, identifier, requirements, incompatibilities): ) def find_matches(self, identifier, requirements, incompatibilities): + """Find all possible candidates that satisfy given constraints. Overridden.""" candidates = sorted( self._iter_matches(identifier, requirements, incompatibilities), key=operator.attrgetter("version"), @@ -169,6 +170,7 @@ def find_matches(self, identifier, requirements, incompatibilities): return candidates def is_satisfied_by(self, requirement, candidate): + """Whether the given requirement can be satisfied by a candidate. Overridden.""" return candidate.version in requirement.specifier def _iter_dependencies(self, candidate): @@ -195,6 +197,7 @@ def _iter_dependencies(self, candidate): yield r def get_dependencies(self, candidate): + """Get dependencies of a candidate. Overridden.""" return list(self._iter_dependencies(candidate)) @@ -264,17 +267,17 @@ def format_resolution(results, as_tree=False): return dependencies -def pypi_simple_repo_in_repos(repos: PypiSimpleRepository): +def pypi_simple_repo_in_repos(repos: utils_pypi.PypiSimpleRepository): """ Return True if simple pypi index_url is present in any of the repos """ - return any(repo.index_url == PYPI_SIMPLE_URL for repo in repos) + return any(repo.index_url == utils_pypi.PYPI_SIMPLE_URL for repo in repos) def get_resolved_dependencies( requirements: List[Requirement], - environment: Environment = None, - repos: List[PypiSimpleRepository] = [], + environment: utils_pypi.Environment = None, + repos: List[utils_pypi.PypiSimpleRepository] = [], as_tree: bool = False, ): """ @@ -283,7 +286,7 @@ def get_resolved_dependencies( parent/children or a nested tree if ``as_tree`` is True """ if repos and not pypi_simple_repo_in_repos(repos): - repos.append(PYPI_PUBLIC_REPO) + repos.append(utils_pypi.PYPI_PUBLIC_REPO) resolver = Resolver( provider=PythonInputProvider(environment, repos), diff --git a/src/python_inspector/resolution.py.LICENSE b/src/python_inspector/resolution.py.LICENSE new file mode 100644 index 00000000..b9077766 --- /dev/null +++ b/src/python_inspector/resolution.py.LICENSE @@ -0,0 +1,13 @@ +Copyright (c) 2018, Tzu-ping Chung + +Permission to use, copy, modify, and distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. From 11583a9e044de650b10a3b32c21f3d2c0752deb8 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Tue, 14 Jun 2022 18:44:20 +0200 Subject: [PATCH 16/19] Do not use PyPI.org JSON API by default * Add new CLI flag "--use-pypi-json-api" to use exclusively the PyPI fast but incorrect JSON API. * Remove pypi_simple_repo_in_repos() from 'python_inspector.resolution' This is not longer needed * Also remove dangerous globals from function and methods arguments: * PythonInputProvider.__init__() where repos was a mutable list * get_resolved_dependencies() where repos was a mutable list * Refactor methods that dealt with PyPI JSON vs. simple by splitting them in two specialized methods Signed-off-by: Philippe Ombredanne --- src/python_inspector/resolution.py | 121 ++++++++++++++++------------ src/python_inspector/resolve_cli.py | 52 ++++++------ tests/test_cli.py | 4 + tests/test_resolution.py | 9 --- 4 files changed, 100 insertions(+), 86 deletions(-) diff --git a/src/python_inspector/resolution.py b/src/python_inspector/resolution.py index cfdaeb4e..6f78bdd4 100644 --- a/src/python_inspector/resolution.py +++ b/src/python_inspector/resolution.py @@ -11,6 +11,7 @@ import operator import os from typing import List +from typing import Sequence import packaging.requirements import packaging.utils @@ -51,9 +52,9 @@ def is_valid_version(parsed_version, requirements, identifier, bad_versions): class PythonInputProvider(AbstractProvider): - def __init__(self, environment=None, repos=[]): + def __init__(self, environment=None, repos=tuple()): self.environment = environment - self.repos = repos + self.repos = repos or [] self.versions_by_package = {} self.dependencies_by_purl = {} @@ -82,21 +83,34 @@ def get_versions_for_package(self, name, repo=None): """ Return a list of versions for a package. """ - versions = [] if repo and self.environment: - for version, package in repo._get_package_versions_map(name).items(): - wheels = package.get_supported_wheels(environment=self.environment) - if list(wheels): - versions.append(version) + return self.get_versions_for_package_from_repo(name, repo) else: - if name not in self.versions_by_package: - api_url = f"https://pypi.org/pypi/{name}/json" - resp = get_response(api_url) - if not resp: - self.versions_by_package[name] = [] - releases = resp.get("releases") or {} - self.versions_by_package[name] = releases.keys() or [] - versions = self.versions_by_package[name] + return self.get_versions_for_package_from_pypi_json_api(name) + + def get_versions_for_package_from_repo(self, name, repo): + """ + Return a list of versions for a package name from a repo + """ + versions = [] + for version, package in repo._get_package_versions_map(name).items(): + wheels = package.get_supported_wheels(environment=self.environment) + if list(wheels): + versions.append(version) + return versions + + def get_versions_for_package_from_pypi_json_api(self, name): + """ + Return a list of versions for a package name from the PyPI.org JSON API + """ + if name not in self.versions_by_package: + api_url = f"https://pypi.org/pypi/{name}/json" + resp = get_response(api_url) + if not resp: + self.versions_by_package[name] = [] + releases = resp.get("releases") or {} + self.versions_by_package[name] = releases.keys() or [] + versions = self.versions_by_package[name] return versions def get_requirements_for_package(self, purl, candidate): @@ -104,32 +118,40 @@ def get_requirements_for_package(self, purl, candidate): Yield requirements for a package. """ if self.repos and self.environment: - wheels = utils_pypi.download_wheel( - name=candidate.name, - version=str(candidate.version), - environment=self.environment, - repos=self.repos, - ) - for wheel in wheels: - deps = list( - PypiWheelHandler.parse(os.path.join(utils_pypi.CACHE_THIRDPARTY_DIR, wheel)) - ) - assert len(deps) == 1 - deps = deps[0].dependencies - for dep in deps: - if dep.scope == "install": - yield packaging.requirements.Requirement(str(dep.extracted_requirement)) + return self.get_requirements_for_package_from_pypi_simple(candidate) else: - if str(purl) not in self.dependencies_by_purl: - api_url = f"https://pypi.org/pypi/{purl.name}/{purl.version}/json" - resp = get_response(api_url) - if not resp: - self.dependencies_by_purl[str(purl)] = [] - info = resp.get("info") or {} - requires_dist = info.get("requires_dist") or [] - self.dependencies_by_purl[str(purl)] = requires_dist - for dependency in self.dependencies_by_purl[str(purl)]: - yield packaging.requirements.Requirement(dependency) + return self.get_requirements_for_package_from_pypi_json_api(purl) + + def get_requirements_for_package_from_pypi_simple(self, candidate): + wheels = utils_pypi.download_wheel( + name=candidate.name, + version=str(candidate.version), + environment=self.environment, + repos=self.repos, + ) + for wheel in wheels: + deps = list( + PypiWheelHandler.parse(os.path.join(utils_pypi.CACHE_THIRDPARTY_DIR, wheel)) + ) + assert len(deps) == 1 + deps = deps[0].dependencies + for dep in deps: + if dep.scope == "install": + yield packaging.requirements.Requirement(str(dep.extracted_requirement)) + + def get_requirements_for_package_from_pypi_json_api(self, purl): + + # if no repos are provided use the incorrect but fast JSON API + if str(purl) not in self.dependencies_by_purl: + api_url = f"https://pypi.org/pypi/{purl.name}/{purl.version}/json" + resp = get_response(api_url) + if not resp: + self.dependencies_by_purl[str(purl)] = [] + info = resp.get("info") or {} + requires_dist = info.get("requires_dist") or [] + self.dependencies_by_purl[str(purl)] = requires_dist + for dependency in self.dependencies_by_purl[str(purl)]: + yield packaging.requirements.Requirement(dependency) def get_candidates(self, all_versions, requirements, identifier, bad_versions, name, extras): """ @@ -267,29 +289,22 @@ def format_resolution(results, as_tree=False): return dependencies -def pypi_simple_repo_in_repos(repos: utils_pypi.PypiSimpleRepository): - """ - Return True if simple pypi index_url is present in any of the repos - """ - return any(repo.index_url == utils_pypi.PYPI_SIMPLE_URL for repo in repos) - - def get_resolved_dependencies( requirements: List[Requirement], environment: utils_pypi.Environment = None, - repos: List[utils_pypi.PypiSimpleRepository] = [], + repos: Sequence[utils_pypi.PypiSimpleRepository] = tuple(), as_tree: bool = False, ): """ Return resolved dependencies of a ``requirements`` list of Requirement for an ``enviroment`` Environment. The resolved dependencies are formatted as - parent/children or a nested tree if ``as_tree`` is True - """ - if repos and not pypi_simple_repo_in_repos(repos): - repos.append(utils_pypi.PYPI_PUBLIC_REPO) + parent/children or a nested tree if ``as_tree`` is True. + Used the provided ``repos`` list of PypiSimpleRepository. + If empty, use instead the PyPI.org JSON API exclusively instead + """ resolver = Resolver( - provider=PythonInputProvider(environment, repos), + provider=PythonInputProvider(environment=environment, repos=repos), reporter=BaseReporter(), ) results = resolver.resolve(requirements=requirements) diff --git a/src/python_inspector/resolve_cli.py b/src/python_inspector/resolve_cli.py index cd2f3306..bf6dd7eb 100644 --- a/src/python_inspector/resolve_cli.py +++ b/src/python_inspector/resolve_cli.py @@ -94,7 +94,13 @@ "--use-cached-index", is_flag=True, hidden=True, - help="Use cached on-disk PyPI package indexes and do not refetch if present.", + help="Use cached on-disk PyPI simple package indexes and do not refetch if present.", +) +@click.option( + "--use-pypi-json-api", + is_flag=True, + help="Use PyPI JSON API to fetch dependency data. Faster but not always correct. " + "--index-url are ignored when this option is active.", ) @click.option( "--debug", @@ -111,6 +117,7 @@ def resolve_dependencies( index_urls, json_output, use_cached_index=False, + use_pypi_json_api=False, debug=TRACE, ): """ @@ -126,16 +133,10 @@ def resolve_dependencies( Error and progress are printed to stderr. - 1) Without an --index-url is provided, this tool uses the PyPI JSON API. - For example, display the results of resolving the dependencies for flask==2.1.2 on screen:: dad --spec "flask==2.1.2" --json - - - 2) If an --index-url is provided, it is used to resolve the dependencies:: - - dad --spec "flask==2.1.2" --index-url https://pypi.org/simple --json - """ click.secho(f"Resolving dependencies...") @@ -158,7 +159,7 @@ def resolve_dependencies( if debug: click.secho("direct_dependencies:") for dep in direct_dependencies: - click.secho(" ", dep) + click.secho(f" {dep}") # create a resolution environments environment = utils_pypi.Environment.from_pyver_and_os( @@ -166,27 +167,28 @@ def resolve_dependencies( ) if debug: - click.secho("environment:", environment) + click.secho(f"environment: {environment}") - # Collect PyPI repos repos = [] - for index_url in index_urls: - index_url = index_url.strip("/") - existing = utils_pypi.DEFAULT_PYPI_REPOS_BY_URL.get(index_url) - if existing: - existing.use_cached_index = use_cached_index - repos.append(existing) - else: - repo = utils_pypi.PypiSimpleRepository( - index_url=index_url, - use_cached_index=use_cached_index, - ) - repos.append(repo) + if not use_pypi_json_api: + # Collect PyPI repos + for index_url in index_urls: + index_url = index_url.strip("/") + existing = utils_pypi.DEFAULT_PYPI_REPOS_BY_URL.get(index_url) + if existing: + existing.use_cached_index = use_cached_index + repos.append(existing) + else: + repo = utils_pypi.PypiSimpleRepository( + index_url=index_url, + use_cached_index=use_cached_index, + ) + repos.append(repo) if debug: click.secho("repos:") for repo in repos: - click.secho(" ", repo) + click.secho(f" {repo}") # resolve dependencies proper requirements, resolved_dependencies = resolve( @@ -230,11 +232,13 @@ def resolve_dependencies( click.secho("done!") -def resolve(direct_dependencies, environment, repos, as_tree=False): +def resolve(direct_dependencies, environment, repos=tuple(), as_tree=False): """ Resolve dependencies given a ``direct_dependencies`` list of DependentPackage and return a tuple of (initial_requirements, resolved_dependencies). + Used the provided ``repos`` list of PypiSimpleRepository. + If empty, use instead the PyPI.org JSON API exclusively. """ requirements = [ diff --git a/tests/test_cli.py b/tests/test_cli.py index 3c4d52d9..595dcb63 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -29,9 +29,13 @@ def test_cli_with_default_urls(): expected_file = test_env.get_test_loc("default-url-expected.json", must_exist=False) specifier = "zipp==3.8.0" + extra_options = [ + "--use-pypi-json-api", + ] check_specs_resolution( specifier=specifier, expected_file=expected_file, + extra_options=extra_options, regen=REGEN_TEST_FIXTURES, ) diff --git a/tests/test_resolution.py b/tests/test_resolution.py index 38cf0098..a895f55d 100644 --- a/tests/test_resolution.py +++ b/tests/test_resolution.py @@ -14,7 +14,6 @@ from python_inspector.resolution import get_resolved_dependencies from python_inspector.resolution import is_valid_version -from python_inspector.resolution import pypi_simple_repo_in_repos from python_inspector.utils_pypi import PYPI_PUBLIC_REPO from python_inspector.utils_pypi import Environment @@ -87,14 +86,6 @@ def test_get_resolved_dependencies_with_tilde_requirement_using_json_api(): ] -def test_pypi_simple_repo_in_repos(): - assert pypi_simple_repo_in_repos(repos=[PYPI_PUBLIC_REPO]) - - -def test_pypi_simple_repo_in_repos_not_present(): - assert not pypi_simple_repo_in_repos(repos=[]) - - def test_is_valid_version(): parsed_version = packaging.version.parse("2.1.2") requirements = {"flask": [Requirement("flask>2.0.0")]} From 89e400adc9644cfc87b51177a0302a8632c4f602 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Tue, 14 Jun 2022 18:55:36 +0200 Subject: [PATCH 17/19] Ensure that tests can run everywhere Signed-off-by: Philippe Ombredanne --- .../pinned-requirements.txt-expected.json | 1 - tests/test_cli.py | 22 ++++++++++++++++++- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/tests/data/pinned-requirements.txt-expected.json b/tests/data/pinned-requirements.txt-expected.json index aef527dd..ffb04c91 100644 --- a/tests/data/pinned-requirements.txt-expected.json +++ b/tests/data/pinned-requirements.txt-expected.json @@ -4,7 +4,6 @@ "tool_homepageurl": "https://github.com/nexB/python-inspector", "tool_version": "0.5.0", "options": [ - "--requirement /home/pombreda/w421/python-inspector/tests/data/pinned-requirements.txt", "--index-url https://pypi.org/simple", "--python-version 38", "--operating-system linux", diff --git a/tests/test_cli.py b/tests/test_cli.py index 595dcb63..508a5481 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -119,11 +119,14 @@ def check_requirements_resolution( ) -def check_json_results(result_file, expected_file, regen=REGEN_TEST_FIXTURES): +def check_json_results(result_file, expected_file, clean=True, regen=REGEN_TEST_FIXTURES): """ Check the ``result_file`` JSON results against the ``expected_file`` expected JSON results. + If ``clean`` is True, remove headers data that can change across runs to + provide stable test resultys. + If ``regen`` is True the expected_file WILL BE overwritten with the new results from ``results_file``. This is convenient for updating tests expectations. @@ -131,6 +134,9 @@ def check_json_results(result_file, expected_file, regen=REGEN_TEST_FIXTURES): with open(result_file) as res: results = json.load(res) + if clean: + clean_results(results) + if regen: with open(expected_file, "w") as reg: json.dump(results, reg, indent=2, separators=(",", ": ")) @@ -139,9 +145,23 @@ def check_json_results(result_file, expected_file, regen=REGEN_TEST_FIXTURES): with open(expected_file) as res: expected = json.load(res) + if clean: + clean_results(expected) + assert results == expected +def clean_results(results): + """ + Return cleaned results removing transient values that can change across test + runs. + """ + headers = results.get("headers", {}) + options = headers.get("options", []) + headers["options"] = [o for o in options if not o.startswith("--requirement")] + return results + + def run_cli(options, cli=resolve_dependencies, expected_rc=0, env=None): """ Run a command line resolution. Return a click.testing.Result object. From b017120bd413feabfa6fcd92512e44632de680fd Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Tue, 14 Jun 2022 18:59:20 +0200 Subject: [PATCH 18/19] Do not check with make Signed-off-by: Philippe Ombredanne --- azure-pipelines.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index bf7c7022..4a299116 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -15,7 +15,6 @@ jobs: test_suites: all: | venv/bin/pytest -n 2 -vvs - make check - template: etc/ci/azure-posix.yml parameters: From 8bdb2574c8667471659d92ee425e423a9cc05bf1 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Wed, 15 Jun 2022 02:43:31 +0530 Subject: [PATCH 19/19] Fix failing tests Signed-off-by: Tushar Goel --- src/python_inspector/resolution.py | 21 ++++++++++++++++- tests/test_resolution.py | 36 +++++++++++++++++++++++++++--- 2 files changed, 53 insertions(+), 4 deletions(-) diff --git a/src/python_inspector/resolution.py b/src/python_inspector/resolution.py index 6f78bdd4..14529dea 100644 --- a/src/python_inspector/resolution.py +++ b/src/python_inspector/resolution.py @@ -51,6 +51,17 @@ def is_valid_version(parsed_version, requirements, identifier, bad_versions): return True +def get_python_version_from_env_tag(python_version: str): + """ + >>> assert get_python_version_from_env_tag("310") == "3.10" + >>> assert get_python_version_from_env_tag("39") == "3.9" + """ + elements = list(python_version) + elements.insert(1, ".") + python_version = "".join(elements) + return python_version + + class PythonInputProvider(AbstractProvider): def __init__(self, environment=None, repos=tuple()): self.environment = environment @@ -215,7 +226,15 @@ def _iter_dependencies(self, candidate): if r.marker is None: yield r else: - if r.marker.evaluate({"extra": ""}): + if r.marker.evaluate( + { + "extra": "", + "python_version": get_python_version_from_env_tag( + self.environment.python_version + ), + "platform_system": self.environment.operating_system.capitalize(), + } + ): yield r def get_dependencies(self, candidate): diff --git a/tests/test_resolution.py b/tests/test_resolution.py index a895f55d..52fb6d97 100644 --- a/tests/test_resolution.py +++ b/tests/test_resolution.py @@ -34,12 +34,34 @@ def test_get_resolved_dependencies_with_flask_and_python_310(): assert as_list == [ "pkg:pypi/click@8.1.3", "pkg:pypi/flask@2.1.2", - "pkg:pypi/importlib-metadata@4.11.4", "pkg:pypi/itsdangerous@2.1.2", "pkg:pypi/jinja2@3.1.2", "pkg:pypi/markupsafe@2.1.1", "pkg:pypi/werkzeug@2.1.2", - "pkg:pypi/zipp@3.8.0", + ] + + +@pytest.mark.online +def test_get_resolved_dependencies_with_flask_and_python_310_windows(): + req = [Requirement("flask==2.1.2")] + results = get_resolved_dependencies( + requirements=req, + environment=Environment( + python_version="310", + operating_system="windows", + ), + repos=[PYPI_PUBLIC_REPO], + as_tree=False, + ) + as_list = [p["package"] for p in results] + assert as_list == [ + "pkg:pypi/click@8.1.3", + "pkg:pypi/colorama@0.4.4", + "pkg:pypi/flask@2.1.2", + "pkg:pypi/itsdangerous@2.1.2", + "pkg:pypi/jinja2@3.1.2", + "pkg:pypi/markupsafe@2.1.1", + "pkg:pypi/werkzeug@2.1.2", ] @@ -64,6 +86,7 @@ def test_get_resolved_dependencies_with_flask_and_python_36(): "pkg:pypi/itsdangerous@2.1.2", "pkg:pypi/jinja2@3.1.2", "pkg:pypi/markupsafe@2.0.1", + "pkg:pypi/typing-extensions@4.2.0", "pkg:pypi/werkzeug@2.1.2", "pkg:pypi/zipp@3.8.0", ] @@ -72,7 +95,14 @@ def test_get_resolved_dependencies_with_flask_and_python_36(): @pytest.mark.online def test_get_resolved_dependencies_with_tilde_requirement_using_json_api(): req = [Requirement("flask~=2.1.2")] - results = get_resolved_dependencies(requirements=req, as_tree=False) + results = get_resolved_dependencies( + requirements=req, + as_tree=False, + environment=Environment( + python_version="38", + operating_system="linux", + ), + ) as_list = [p["package"] for p in results] assert as_list == [ "pkg:pypi/click@8.1.3",