diff --git a/src/python_inspector/resolution.py b/src/python_inspector/resolution.py index 6d275957..ce22643c 100644 --- a/src/python_inspector/resolution.py +++ b/src/python_inspector/resolution.py @@ -101,6 +101,17 @@ def get_requirements_from_distribution( return list(get_requirements_from_dependencies(dependencies=deps[0].dependencies)) +def get_environment_marker_from_environment(environment): + return { + "extra": "", + "python_version": get_python_version_from_env_tag( + python_version=environment.python_version + ), + "platform_system": environment.operating_system.capitalize(), + "sys_platform": environment.operating_system, + } + + def is_requirements_file_in_setup_files(setup_files: List[str]) -> bool: """ Return True if the string ``requirements.txt`` is found in any of the ``setup_files`` location @@ -220,6 +231,7 @@ def remove_extras(identifier: str) -> str: class PythonInputProvider(AbstractProvider): def __init__(self, environment=None, repos=tuple()): self.environment = environment + self.environment_marker = get_environment_marker_from_environment(environment) self.repos = repos or [] self.versions_by_package = {} self.dependencies_by_purl = {} @@ -484,16 +496,7 @@ def _iter_dependencies(self, candidate: Candidate) -> Generator[Requirement, Non if r.marker is None: yield r else: - if r.marker.evaluate( - { - "extra": "", - "python_version": get_python_version_from_env_tag( - python_version=self.environment.python_version - ), - "platform_system": self.environment.operating_system.capitalize(), - "sys_platform": self.environment.operating_system, - } - ): + if r.marker.evaluate(self.environment_marker): yield r def get_dependencies(self, candidate: Candidate) -> List[Requirement]: diff --git a/src/python_inspector/resolve_cli.py b/src/python_inspector/resolve_cli.py index a87286d5..9e7745c7 100644 --- a/src/python_inspector/resolve_cli.py +++ b/src/python_inspector/resolve_cli.py @@ -11,6 +11,7 @@ import json import os +from typing import Dict from typing import List import click @@ -24,6 +25,7 @@ from python_inspector import utils from python_inspector import utils_pypi from python_inspector.cli_utils import FileOptionType +from python_inspector.resolution import get_environment_marker_from_environment from python_inspector.resolution import get_resolved_dependencies TRACE = False @@ -351,7 +353,13 @@ def resolve( If empty, use instead the PyPI.org JSON API exclusively. """ - requirements = list(get_requirements_from_direct_dependencies(direct_dependencies)) + environment_marker = get_environment_marker_from_environment(environment) + + requirements = list( + get_requirements_from_direct_dependencies( + direct_dependencies=direct_dependencies, environment_marker=environment_marker + ) + ) resolved_dependencies = get_resolved_dependencies( requirements=requirements, @@ -369,7 +377,7 @@ def resolve( def get_requirements_from_direct_dependencies( - direct_dependencies: List[DependentPackage], + direct_dependencies: List[DependentPackage], environment_marker: Dict ) -> List[Requirement]: for dependency in direct_dependencies: # FIXME We are skipping editable requirements @@ -377,7 +385,12 @@ def get_requirements_from_direct_dependencies( # https://github.com/nexB/python-inspector/issues/41 if not can_process_dependent_package(dependency): continue - yield Requirement(requirement_string=dependency.extracted_requirement) + req = Requirement(requirement_string=dependency.extracted_requirement) + if req.marker is None: + yield req + else: + if req.marker.evaluate(environment_marker): + yield req def write_output(headers, requirements, resolved_dependencies, json_output, pdt_output=False): diff --git a/tests/data/environment-marker-test-requirements.txt b/tests/data/environment-marker-test-requirements.txt new file mode 100644 index 00000000..41141746 --- /dev/null +++ b/tests/data/environment-marker-test-requirements.txt @@ -0,0 +1,7 @@ +click>6,<6.8 +Flask==1.0 +itsdangerous<0.25 +license-expression ; platform_system == "Windows" +Jinja2==2.11.3 +MarkupSafe==1.0 +Werkzeug==0.15.3 \ No newline at end of file diff --git a/tests/data/pdt-expected.json b/tests/data/environment-marker-test-requirements.txt-expected.json similarity index 100% rename from tests/data/pdt-expected.json rename to tests/data/environment-marker-test-requirements.txt-expected.json diff --git a/tests/data/frozen-requirements-pdt-expected.json b/tests/data/frozen-requirements.txt-expected.json similarity index 100% rename from tests/data/frozen-requirements-pdt-expected.json rename to tests/data/frozen-requirements.txt-expected.json diff --git a/tests/data/pdt-requirements.txt-expected.json b/tests/data/pdt-requirements.txt-expected.json new file mode 100644 index 00000000..dd085dd6 --- /dev/null +++ b/tests/data/pdt-requirements.txt-expected.json @@ -0,0 +1,40 @@ +[ + { + "key": "flask", + "package_name": "flask", + "installed_version": "1.0", + "dependencies": [ + { + "key": "click", + "package_name": "click", + "installed_version": "6.7", + "dependencies": [] + }, + { + "key": "itsdangerous", + "package_name": "itsdangerous", + "installed_version": "0.24", + "dependencies": [] + }, + { + "key": "jinja2", + "package_name": "jinja2", + "installed_version": "2.11.3", + "dependencies": [ + { + "key": "markupsafe", + "package_name": "markupsafe", + "installed_version": "1.0", + "dependencies": [] + } + ] + }, + { + "key": "werkzeug", + "package_name": "werkzeug", + "installed_version": "0.15.3", + "dependencies": [] + } + ] + } +] \ No newline at end of file diff --git a/tests/data/pinned-pdt-requirements.txt b/tests/data/pinned-pdt-requirements.txt new file mode 100644 index 00000000..f3c5ff11 --- /dev/null +++ b/tests/data/pinned-pdt-requirements.txt @@ -0,0 +1,29 @@ +aboutcode-toolkit==7.0.2 +attrs==21.4.0 +beautifulsoup4==4.11.1 +certifi==2022.5.18.1 +charset-normalizer==2.0.12 +click==8.0.4 +colorama==0.4.4 +commoncode==30.2.0 +dparse2==0.6.1 +idna==3.3 +importlib-metadata==4.8.3 +intbitset==3.0.1 +packageurl-python==0.9.9 +packaging==21.3 +-e git+https://github.com/nexB/python-inspector@18baae17824d6bacb4b1d519b10a0d0e50775884#egg=python_inspector +pip-requirements-parser==31.2.0 +pkginfo2==30.0.0 +pyparsing==3.0.9 +PyYAML==6.0 +requests==2.27.1 +resolvelib==0.8.1 +saneyaml==0.5.2 +soupsieve==2.3.2.post1 +text-unidecode==1.3 +toml==0.10.2 +typing==3.6.6 +typing_extensions==4.1.1 +urllib3==1.26.9 +zipp==3.6.0 diff --git a/tests/data/pinned-requirements-pdt-expected.json b/tests/data/pinned-pdt-requirements.txt-expected.json similarity index 100% rename from tests/data/pinned-requirements-pdt-expected.json rename to tests/data/pinned-pdt-requirements.txt-expected.json diff --git a/tests/data/single-url-except-simple-expected.json b/tests/data/single-url-except-simple-expected.json index 46016c89..e88e6366 100644 --- a/tests/data/single-url-except-simple-expected.json +++ b/tests/data/single-url-except-simple-expected.json @@ -37,18 +37,18 @@ "sdist_url": "https://files.pythonhosted.org/packages/59/87/84326af34517fca8c58418d148f2403df25303e02736832403587318e9e8/click-8.1.3.tar.gz" }, { - "package": "pkg:pypi/flask@2.2.1", + "package": "pkg:pypi/flask@2.2.2", "dependencies": [ "pkg:pypi/click@8.1.3", "pkg:pypi/importlib-metadata@4.12.0", "pkg:pypi/itsdangerous@2.1.2", "pkg:pypi/jinja2@3.1.2", - "pkg:pypi/werkzeug@2.2.1" + "pkg:pypi/werkzeug@2.2.2" ], "wheel_urls": [ - "https://files.pythonhosted.org/packages/3c/96/6c896f80f466b7f5e2cfd6d632fe5b0464dcb412757c595a663e59589a93/Flask-2.2.1-py3-none-any.whl" + "https://files.pythonhosted.org/packages/0f/43/15f4f9ab225b0b25352412e8daa3d0e3d135fcf5e127070c74c3632c8b4c/Flask-2.2.2-py3-none-any.whl" ], - "sdist_url": "https://files.pythonhosted.org/packages/4b/4f/50888944490f2263bd70171e8298c9626675fd3dfd750694a7beaa3484fb/Flask-2.2.1.tar.gz" + "sdist_url": "https://files.pythonhosted.org/packages/69/b6/53cfa30eed5aa7343daff36622843688ba8c6fe9829bb2b92e193ab1163f/Flask-2.2.2.tar.gz" }, { "package": "pkg:pypi/importlib-metadata@4.12.0", @@ -87,14 +87,14 @@ "sdist_url": "https://files.pythonhosted.org/packages/1d/97/2288fe498044284f39ab8950703e88abbac2abbdf65524d576157af70556/MarkupSafe-2.1.1.tar.gz" }, { - "package": "pkg:pypi/werkzeug@2.2.1", + "package": "pkg:pypi/werkzeug@2.2.2", "dependencies": [ "pkg:pypi/markupsafe@2.1.1" ], "wheel_urls": [ - "https://files.pythonhosted.org/packages/c0/93/d6d60870e47162ea6a1bbdd787649eea776b2a70618dd66ed87cb2238543/Werkzeug-2.2.1-py3-none-any.whl" + "https://files.pythonhosted.org/packages/c8/27/be6ddbcf60115305205de79c29004a0c6bc53cec814f733467b1bb89386d/Werkzeug-2.2.2-py3-none-any.whl" ], - "sdist_url": "https://files.pythonhosted.org/packages/32/19/a92cdbd9fb795928dfca1031278ae8a7f051e78a2c057c224ad2d4cdd95e/Werkzeug-2.2.1.tar.gz" + "sdist_url": "https://files.pythonhosted.org/packages/f8/c1/1c8e539f040acd80f844c69a5ef8e2fccdf8b442dabb969e497b55d544e1/Werkzeug-2.2.2.tar.gz" }, { "package": "pkg:pypi/zipp@3.8.1", diff --git a/tests/test_cli.py b/tests/test_cli.py index c2a69eaa..6842066f 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -45,7 +45,7 @@ def test_cli_with_default_urls(): @pytest.mark.online def test_pdt_output(): requirements_file = test_env.get_test_loc("pdt-requirements.txt") - expected_file = test_env.get_test_loc("pdt-expected.json", must_exist=False) + expected_file = test_env.get_test_loc("pdt-requirements.txt-expected.json", must_exist=False) extra_options = [] check_requirements_resolution( requirements_file=requirements_file, @@ -58,8 +58,10 @@ def test_pdt_output(): @pytest.mark.online def test_pdt_output_with_pinned_requirements(): - requirements_file = test_env.get_test_loc("pinned-requirements.txt") - expected_file = test_env.get_test_loc("pinned-requirements-pdt-expected.json", must_exist=False) + requirements_file = test_env.get_test_loc("pinned-pdt-requirements.txt") + expected_file = test_env.get_test_loc( + "pinned-pdt-requirements.txt-expected.json", must_exist=False + ) extra_options = [] check_requirements_resolution( requirements_file=requirements_file, @@ -73,7 +75,7 @@ def test_pdt_output_with_pinned_requirements(): @pytest.mark.online def test_pdt_output_with_frozen_requirements(): requirements_file = test_env.get_test_loc("frozen-requirements.txt") - expected_file = test_env.get_test_loc("frozen-requirements-pdt-expected.json", must_exist=False) + expected_file = test_env.get_test_loc("frozen-requirements.txt-expected.json", must_exist=False) extra_options = [] check_requirements_resolution( requirements_file=requirements_file, @@ -137,6 +139,27 @@ def test_cli_with_multiple_index_url_and_tilde_req(): ) +@pytest.mark.online +def test_cli_with_environment_marker_and_complex_ranges(): + requirements_file = test_env.get_test_loc("environment-marker-test-requirements.txt") + expected_file = test_env.get_test_loc( + "environment-marker-test-requirements.txt-expected.json", must_exist=False + ) + extra_options = [ + "--operating-system", + "linux", + "--python-version", + "37", + ] + check_requirements_resolution( + requirements_file=requirements_file, + expected_file=expected_file, + extra_options=extra_options, + pdt_output=True, + regen=REGEN_TEST_FIXTURES, + ) + + @pytest.mark.online def test_cli_with_multiple_index_url_and_tilde_req_with_max_rounds(): expected_file = test_env.get_test_loc("tilde_req-expected.json", must_exist=False) @@ -345,13 +368,23 @@ def test_get_requirements_from_direct_dependencies(): ) ] - requirements = [str(r) for r in get_requirements_from_direct_dependencies(direct_dependencies)] + requirements = [ + str(r) + for r in get_requirements_from_direct_dependencies( + direct_dependencies=direct_dependencies, environment_marker={} + ) + ] assert requirements == ["django>=1.11.11"] def test_get_requirements_from_direct_dependencies_with_empty_list(): - assert list(get_requirements_from_direct_dependencies(direct_dependencies=[])) == [] + assert ( + list( + get_requirements_from_direct_dependencies(direct_dependencies=[], environment_marker={}) + ) + == [] + ) def test_get_requirements_from_direct_dependencies_with_editable_requirements(): @@ -378,6 +411,11 @@ def test_get_requirements_from_direct_dependencies_with_editable_requirements(): ) ] - requirements = [str(r) for r in get_requirements_from_direct_dependencies(direct_dependencies)] + requirements = [ + str(r) + for r in get_requirements_from_direct_dependencies( + direct_dependencies=direct_dependencies, environment_marker={} + ) + ] assert requirements == [] diff --git a/tests/test_resolution.py b/tests/test_resolution.py index df73233e..7523861b 100644 --- a/tests/test_resolution.py +++ b/tests/test_resolution.py @@ -40,7 +40,7 @@ def test_get_resolved_dependencies_with_flask_and_python_310(): "pkg:pypi/itsdangerous@2.1.2", "pkg:pypi/jinja2@3.1.2", "pkg:pypi/markupsafe@2.1.1", - "pkg:pypi/werkzeug@2.2.1", + "pkg:pypi/werkzeug@2.2.2", ] @@ -65,7 +65,7 @@ def test_get_resolved_dependencies_with_flask_and_python_310_windows(): "pkg:pypi/itsdangerous@2.1.2", "pkg:pypi/jinja2@3.1.2", "pkg:pypi/markupsafe@2.1.1", - "pkg:pypi/werkzeug@2.2.1", + "pkg:pypi/werkzeug@2.2.2", ] @@ -118,7 +118,7 @@ def test_get_resolved_dependencies_with_tilde_requirement_using_json_api(): "pkg:pypi/itsdangerous@2.1.2", "pkg:pypi/jinja2@3.1.2", "pkg:pypi/markupsafe@2.1.1", - "pkg:pypi/werkzeug@2.2.1", + "pkg:pypi/werkzeug@2.2.2", "pkg:pypi/zipp@3.8.1", ] @@ -145,7 +145,7 @@ def test_without_supported_wheels(): "pkg:pypi/hyperlink@21.0.0", "pkg:pypi/idna@3.3", "pkg:pypi/pycparser@2.21", - "pkg:pypi/setuptools@63.4.1", + "pkg:pypi/setuptools@65.0.0", "pkg:pypi/txaio@22.2.1", ]