From 536df603bfb470fedce1b21f3b3daa5a874e8fc0 Mon Sep 17 00:00:00 2001 From: Lorenzo Pirritano Date: Sun, 15 Dec 2024 21:01:55 +0100 Subject: [PATCH 01/10] poetry deps w/ python version and platform and multiple constraints dependencies --- grayskull/strategy/parse_poetry_version.py | 116 +++++++++++++++++ grayskull/strategy/py_toml.py | 65 ++++++++-- tests/test_parse_poetry_version.py | 61 ++++++++- tests/test_py_toml.py | 143 ++++++++++++++++++++- 4 files changed, 369 insertions(+), 16 deletions(-) diff --git a/grayskull/strategy/parse_poetry_version.py b/grayskull/strategy/parse_poetry_version.py index f5066dbad..e825fd752 100644 --- a/grayskull/strategy/parse_poetry_version.py +++ b/grayskull/strategy/parse_poetry_version.py @@ -236,3 +236,119 @@ def encode_poetry_version(poetry_specifier: str) -> str: conda_clauses.append(poetry_clause) return ",".join(conda_clauses) + + +def encode_poetry_platform_to_selector_item(poetry_platform: str) -> str: + """ + Encodes Poetry Platform specifier as a Conda selector. + + Example: "darwin" => "osx" + """ + + platform_selectors = {"windows": "win", "linux": "linux", "darwin": "osx"} + poetry_platform = poetry_platform.lower().strip() + if poetry_platform in platform_selectors: + return platform_selectors[poetry_platform] + else: # unknown + return "" + + +def encode_poetry_python_version_to_selector_item(poetry_specifier: str) -> str: + """ + Encodes Poetry Python version specifier as a Conda selector. + + Example: ">=3.8,<3.12" => "py>=38 or py<312" + + # handle exact version specifiers correctly + >>> encode_poetry_python_version_to_selector_item("3.8") + "py==38" + >>> encode_poetry_python_version_to_selector_item("==3.8") + "py==38" + >>> encode_poetry_python_version_to_selector_item("!=3.8") + "py!=38" + + # handle caret operator correctly + >>> encode_poetry_python_version_to_selector_item("^3.10") + # renders '>=3.10.0,<4.0.0' + "py>=310 or py<4" + + # handle tilde operator correctly + >>> encode_poetry_python_version_to_selector_item("~3.10") + # renders '>=3.10.0,<3.11.0' + "py>=310 or py<311" + """ + + if not poetry_specifier: + return "" + + version_specifier = encode_poetry_version(poetry_specifier) + + conda_clauses = version_specifier.split(",") + + conda_selectors = [] + for conda_clause in conda_clauses: + operator, version = parse_python_version(conda_clause) + version_selector = version.replace(".", "") + conda_selectors.append(f"py{operator}{version_selector}") + selectors = " or ".join(conda_selectors) + return selectors + + +def parse_python_version(selector: str): + """ + Return operator and normalized version from a version selector + + Examples: + ">=3.8" -> ">=", "3.8" + ">=3.8.0" -> ">=", "3.8" + "<4.0.0" -> "<", "4" + "3.12" -> "==", 3.12" + "=3.8" -> "==", "3.8" + + The version is normalized to "major.minor" (drop patch if present) + or "major" if minor is 0 + """ + # Regex to split operator and version + pattern = r"^(?P\^|~|>=|<=|!=|==|>|<|=)?(?P\d+(\.\d+){0,2})$" + match = re.match(pattern, selector) + if not match: + raise ValueError(f"Invalid version selector: {selector}") + + # Extract operator and version + operator = match.group("operator") + # Default to "==" if no operator is provided or "=" + operator = "==" if operator in {None, "="} else operator + version = match.group("version") + + # Normalize version to major.minor (drop patch if present) + # or major if minor is 0 + # + # Some timing to justify this choice: + # Using str.endswith: 0.065 seconds + # Using regex : 0.088 seconds + # Using replace : 0.088 seconds + # Using split : 0.085-0.126 seconds + if version.endswith(".0.0"): + normalized_version = version[:-4] + elif version.endswith(".0"): + normalized_version = version[:-2] + else: + normalized_version = version + return operator, normalized_version + + +def combine_conda_selectors(python_selector: str, platform_selector: str): + """ + Combine selectors based on presence + """ + if python_selector and platform_selector: + if " or " in python_selector: + python_selector = f"({python_selector})" + selector = f"{python_selector} and {platform_selector}" + elif python_selector: + selector = f"{python_selector}" + elif platform_selector: + selector = f"{platform_selector}" + else: + selector = "" + return f" # [{selector}]" if selector else "" diff --git a/grayskull/strategy/py_toml.py b/grayskull/strategy/py_toml.py index f2e8b2329..fd53e817b 100644 --- a/grayskull/strategy/py_toml.py +++ b/grayskull/strategy/py_toml.py @@ -1,9 +1,15 @@ import sys from collections import defaultdict +from collections.abc import Iterator from functools import singledispatch from pathlib import Path -from grayskull.strategy.parse_poetry_version import encode_poetry_version +from grayskull.strategy.parse_poetry_version import ( + combine_conda_selectors, + encode_poetry_platform_to_selector_item, + encode_poetry_python_version_to_selector_item, + encode_poetry_version, +) from grayskull.utils import nested_dict if sys.version_info >= (3, 11): @@ -17,35 +23,68 @@ class InvalidPoetryDependency(BaseException): @singledispatch -def get_constrained_dep(dep_spec: str | dict, dep_name: str) -> str: +def get_constrained_dep(dep_spec: list | str | dict, dep_name: str) -> str: raise InvalidPoetryDependency( - "Expected Poetry dependency specification to be of type str or dict, " + "Expected Poetry dependency specification to be of type list, str or dict, " f"received {type(dep_spec).__name__}" ) @get_constrained_dep.register -def __get_constrained_dep_dict(dep_spec: dict, dep_name: str) -> str: +def __get_constrained_dep_dict( + dep_spec: dict, dep_name: str +) -> Iterator[str, None, None]: + """ + Yield a dependency entry in conda format from a Poetry entry + with version, python version, and platform + + Example: + dep_spec: + {"version": "^1.5", "python": ">=3.8,<3.12", "platform": "darwin"}, + dep_name: + "pandas", + result yield: + "pandas >=1.5.0,<2.0.0 # [(py>=38 or py<312) and osx]" + """ conda_version = encode_poetry_version(dep_spec.get("version", "")) - return f"{dep_name} {conda_version}".strip() + if conda_version: + conda_version = f" {conda_version}" + python_selector = encode_poetry_python_version_to_selector_item( + dep_spec.get("python", "") + ) + platform_selector = encode_poetry_platform_to_selector_item( + dep_spec.get("platform", "") + ) + conda_selector = combine_conda_selectors(python_selector, platform_selector) + yield f"{dep_name}{conda_version}{conda_selector}".strip() @get_constrained_dep.register -def __get_constrained_dep_str(dep_spec: str, dep_name: str) -> str: +def __get_constrained_dep_str( + dep_spec: str, dep_name: str +) -> Iterator[str, None, None]: conda_version = encode_poetry_version(dep_spec) - return f"{dep_name} {conda_version}" + yield f"{dep_name} {conda_version}" + + +@get_constrained_dep.register +def __get_constrained_dep_list( + dep_spec_list: list, dep_name: str +) -> Iterator[str, None, None]: + for dep_spec in dep_spec_list: + yield from get_constrained_dep(dep_spec, dep_name) def encode_poetry_deps(poetry_deps: dict) -> tuple[list, list]: run = [] run_constrained = [] for dep_name, dep_spec in poetry_deps.items(): - constrained_dep = get_constrained_dep(dep_spec, dep_name) - try: - assert dep_spec.get("optional", False) - run_constrained.append(constrained_dep) - except (AttributeError, AssertionError): - run.append(constrained_dep) + for constrained_dep in get_constrained_dep(dep_spec, dep_name): + try: + assert dep_spec.get("optional", False) + run_constrained.append(constrained_dep) + except (AttributeError, AssertionError): + run.append(constrained_dep) return run, run_constrained diff --git a/tests/test_parse_poetry_version.py b/tests/test_parse_poetry_version.py index 96fdeb1bb..60cb92d7a 100644 --- a/tests/test_parse_poetry_version.py +++ b/tests/test_parse_poetry_version.py @@ -2,7 +2,12 @@ import pytest -from grayskull.strategy.parse_poetry_version import InvalidVersion, parse_version +from grayskull.strategy.parse_poetry_version import ( + InvalidVersion, + combine_conda_selectors, + encode_poetry_python_version_to_selector_item, + parse_version, +) @pytest.mark.parametrize( @@ -11,3 +16,57 @@ def test_parse_version_failure(invalid_version): with pytest.raises(InvalidVersion): parse_version(invalid_version) + + +@pytest.mark.parametrize( + "poetry_python_specifier, exp_selector_item", + [ + ("", ""), + (">=3.5", "py>=35"), + (">=3.6", "py>=36"), + (">3.7", "py>37"), + ("<=3.7", "py<=37"), + ("<3.7", "py<37"), + ("3.10", "py==310"), + ("=3.10", "py==310"), + ("==3.10", "py==310"), + ("==3", "py==3"), + (">3.12", "py>312"), + ("!=3.7", "py!=37"), + # multiple specifiers + (">3.7,<3.12", "py>37 or py<312"), + # poetry specifiers + ("^3.10", "py>=310 or py<4"), + ("~3.10", "py>=310 or py<311"), + # PEP 440 not common specifiers + # ("~=3.7", "<37", "<37"), + # ("3.*", "<37", "<37"), + # ("!=3.*", "<37", "<37"), + ], +) +def test_encode_poetry_python_version_to_selector_item( + poetry_python_specifier, exp_selector_item +): + assert exp_selector_item == encode_poetry_python_version_to_selector_item( + poetry_python_specifier + ) + + +@pytest.mark.parametrize( + "python_selector, platform_selector, exp_conda_selector_content", + [ + ("py>=38 or py<312", "osx", "(py>=38 or py<312) and osx"), + ("py>=38 or py<312", "", "py>=38 or py<312"), + ("", "osx", "osx"), + ("py>=38", "", "py>=38"), + ("py<310", "win", "py<310 and win"), + ], +) +def test_combine_conda_selectors( + python_selector, platform_selector, exp_conda_selector_content +): + conda_selector = combine_conda_selectors(python_selector, platform_selector) + expected = ( + f" # [{exp_conda_selector_content}]" if exp_conda_selector_content else "" + ) + assert conda_selector == expected diff --git a/tests/test_py_toml.py b/tests/test_py_toml.py index 1c3cdf9c0..2f0a823ac 100644 --- a/tests/test_py_toml.py +++ b/tests/test_py_toml.py @@ -88,13 +88,152 @@ def test_poetry_langchain_snapshot(tmpdir): def test_poetry_get_constrained_dep_version_not_present(): assert ( - get_constrained_dep( - {"git": "https://codeberg.org/hjacobs/pytest-kind.git"}, "pytest-kind" + next( + get_constrained_dep( + {"git": "https://codeberg.org/hjacobs/pytest-kind.git"}, "pytest-kind" + ) ) == "pytest-kind" ) +def test_poetry_get_constrained_dep_version_string(): + assert next(get_constrained_dep(">=2022.8.2", "s3fs")) == "s3fs >=2022.8.2" + + +def test_poetry_get_constrained_dep_tilde_version_string(): + assert next(get_constrained_dep("~0.21.0", "s3fs")) == "s3fs >=0.21.0,<0.22.0" + + +def test_poetry_get_constrained_dep_caret_version_string(): + assert next(get_constrained_dep("^1.24.0", "numpy")) == "numpy >=1.24.0,<2.0.0" + + +def test_poetry_get_constrained_dep_no_version_only_python(): + assert ( + next( + get_constrained_dep( + {"python": ">=3.8"}, + "validators", + ) + ) + == "validators # [py>=38]" + ) + + +def test_poetry_get_constrained_dep_no_version_only_python_version_and_platform(): + assert ( + next( + get_constrained_dep( + {"python": ">=3.8", "platform": "darwin"}, + "validators", + ) + ) + == "validators # [py>=38 and osx]" + ) + + +def test_poetry_get_constrained_dep_caret_version_python_version_min_max_and_platform(): + assert ( + next( + get_constrained_dep( + {"version": "^1.5", "python": ">=3.8,<3.12", "platform": "darwin"}, + "pandas", + ) + ) + == "pandas >=1.5.0,<2.0.0 # [(py>=38 or py<312) and osx]" + ) + + +def test_poetry_get_constrained_dep_no_version_only_platform(): + assert ( + next( + get_constrained_dep( + {"platform": "darwin"}, + "validators", + ) + ) + == "validators # [osx]" + ) + + +def test_poetry_get_constrained_dep_caret_version_python_minimum_version(): + assert ( + next( + get_constrained_dep( + {"version": "~0.21.0", "python": ">=3.8"}, + "validators", + ) + ) + == "validators >=0.21.0,<0.22.0 # [py>=38]" + ) + + +def test_poetry_get_constrained_dep_caret_version_python_maximum_version(): + assert ( + next( + get_constrained_dep( + [{"version": "^1.24.0", "python": "<3.10"}], + "numpy", + ) + ) + == "numpy >=1.24.0,<2.0.0 # [py<310]" + ) + + +def test_poetry_get_constrained_dep_multiple_constraints_dependencies_with_platform(): + assert list( + get_constrained_dep( + [ + {"version": "^1.24.0", "python": "<3.10"}, + {"version": "^1.26.0", "python": ">=3.10"}, + {"version": "^1.26.0", "python": ">=3.8,<3.10", "platform": "darwin"}, + ], + "numpy", + ) + ) == [ + "numpy >=1.24.0,<2.0.0 # [py<310]", + "numpy >=1.26.0,<2.0.0 # [py>=310]", + "numpy >=1.26.0,<2.0.0 # [(py>=38 or py<310) and osx]", + ] + + +def test_poetry_get_constrained_dep_multiple_constraints_dependencies_ersilia(): + assert ( + next( + get_constrained_dep( + [{"version": "~0.21.0", "python": ">=3.8"}], + "validators", + ) + ) + == "validators >=0.21.0,<0.22.0 # [py>=38]" + ) + + +def test_poetry_get_constrained_dep_multiple_constraints_dependencies_xypattern(): + assert list( + get_constrained_dep( + [ + {"version": "^1.24.0", "python": "<3.10"}, + {"version": "^1.26.0", "python": ">=3.10"}, + ], + "numpy", + ) + ) == ["numpy >=1.24.0,<2.0.0 # [py<310]", "numpy >=1.26.0,<2.0.0 # [py>=310]"] + + +def test_poetry_get_constrained_dep_multiple_constraints_dependencies_nannyml(): + assert ( + next( + get_constrained_dep( + [{"version": "^1.5", "python": ">=3.8,<3.12"}], + "pandas", + ) + ) + == "pandas >=1.5.0,<2.0.0 # [py>=38 or py<312]" + ) + + def test_poetry_entrypoints(): poetry = { "requirements": {"host": ["setuptools"], "run": ["python"]}, From 1a3a85f22a844f2a0eebeffd7dbf15a92ea24a75 Mon Sep 17 00:00:00 2001 From: Lorenzo Pirritano Date: Sun, 15 Dec 2024 21:45:43 +0100 Subject: [PATCH 02/10] add regr test for databricks-sql-connector --- tests/test_py_toml.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/test_py_toml.py b/tests/test_py_toml.py index 2f0a823ac..5e8be26ee 100644 --- a/tests/test_py_toml.py +++ b/tests/test_py_toml.py @@ -234,6 +234,21 @@ def test_poetry_get_constrained_dep_multiple_constraints_dependencies_nannyml(): ) +def test_poetry_get_constrained_dep_mult_constraints_deps_databricks_sql_connector(): + assert list( + get_constrained_dep( + [ + {"version": ">=6.0.0", "python": ">=3.7,<3.11"}, + {"version": ">=10.0.1", "python": ">=3.11"}, + ], + "pyarrow", + ) + ) == [ + "pyarrow >=6.0.0 # [py>=37 or py<311]", + "pyarrow >=10.0.1 # [py>=311]", + ] + + def test_poetry_entrypoints(): poetry = { "requirements": {"host": ["setuptools"], "run": ["python"]}, From 8ed0792bcc01420700754d3c12a3ed2cdbbc0720 Mon Sep 17 00:00:00 2001 From: Lorenzo Pirritano Date: Sun, 15 Dec 2024 22:17:41 +0100 Subject: [PATCH 03/10] fix parse_python_version (drop patch), w/ tests --- grayskull/strategy/parse_poetry_version.py | 20 +++++--------------- tests/test_parse_poetry_version.py | 18 ++++++++++++++++++ 2 files changed, 23 insertions(+), 15 deletions(-) diff --git a/grayskull/strategy/parse_poetry_version.py b/grayskull/strategy/parse_poetry_version.py index e825fd752..2e19a411b 100644 --- a/grayskull/strategy/parse_poetry_version.py +++ b/grayskull/strategy/parse_poetry_version.py @@ -320,21 +320,11 @@ def parse_python_version(selector: str): operator = "==" if operator in {None, "="} else operator version = match.group("version") - # Normalize version to major.minor (drop patch if present) - # or major if minor is 0 - # - # Some timing to justify this choice: - # Using str.endswith: 0.065 seconds - # Using regex : 0.088 seconds - # Using replace : 0.088 seconds - # Using split : 0.085-0.126 seconds - if version.endswith(".0.0"): - normalized_version = version[:-4] - elif version.endswith(".0"): - normalized_version = version[:-2] - else: - normalized_version = version - return operator, normalized_version + # Split into major, minor, and discard the rest (patch or additional parts) + major, minor, *_ = version.split(".") + + # Return only major if minor is "0", otherwise return major.minor + return operator, major if minor == "0" else f"{major}.{minor}" def combine_conda_selectors(python_selector: str, platform_selector: str): diff --git a/tests/test_parse_poetry_version.py b/tests/test_parse_poetry_version.py index 60cb92d7a..56c8d0a3f 100644 --- a/tests/test_parse_poetry_version.py +++ b/tests/test_parse_poetry_version.py @@ -6,6 +6,7 @@ InvalidVersion, combine_conda_selectors, encode_poetry_python_version_to_selector_item, + parse_python_version, parse_version, ) @@ -52,6 +53,23 @@ def test_encode_poetry_python_version_to_selector_item( ) +@pytest.mark.parametrize( + "python_version, exp_operator_version", + [ + (">=3.8", (">=", "3.8")), + (">=3.8.0", (">=", "3.8")), + ("<4.0.0", ("<", "4")), + ("3.12", ("==", "3.12")), + ("=3.8", ("==", "3.8")), + ("=3.8.1", ("==", "3.8")), + ("3.8.1", ("==", "3.8")), + ], +) +def test_parse_python_version(python_version, exp_operator_version): + operator, version = parse_python_version(python_version) + assert (operator, version) == exp_operator_version + + @pytest.mark.parametrize( "python_selector, platform_selector, exp_conda_selector_content", [ From c04e184cb76a61e3408ccd6c1b30785c2ba47d59 Mon Sep 17 00:00:00 2001 From: Lorenzo Pirritano Date: Tue, 17 Dec 2024 10:56:17 +0100 Subject: [PATCH 04/10] add support 1 digit to parse_python_version --- grayskull/strategy/parse_poetry_version.py | 27 ++++++++++++++++++++-- tests/test_parse_poetry_version.py | 1 + 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/grayskull/strategy/parse_poetry_version.py b/grayskull/strategy/parse_poetry_version.py index 2e19a411b..cfff04480 100644 --- a/grayskull/strategy/parse_poetry_version.py +++ b/grayskull/strategy/parse_poetry_version.py @@ -260,6 +260,8 @@ def encode_poetry_python_version_to_selector_item(poetry_specifier: str) -> str: Example: ">=3.8,<3.12" => "py>=38 or py<312" # handle exact version specifiers correctly + >>> encode_poetry_python_version_to_selector_item("3") + 'py==3' >>> encode_poetry_python_version_to_selector_item("3.8") "py==38" >>> encode_poetry_python_version_to_selector_item("==3.8") @@ -299,17 +301,33 @@ def parse_python_version(selector: str): Return operator and normalized version from a version selector Examples: + ">=3" -> ">=", "3" + ">=3.0" -> ">=", "3" ">=3.8" -> ">=", "3.8" ">=3.8.0" -> ">=", "3.8" "<4.0.0" -> "<", "4" "3.12" -> "==", 3.12" "=3.8" -> "==", "3.8" + ">=3.8.0.1" -> ">=", "3.8" + + >>> parse_python_version(">=3.8") + ('>=', '3.8') + >>> parse_python_version("3.12") + ('==', '3.12') + >>> parse_python_version("<4.0.0") + ('<', '4') + >>> parse_python_version(">=3") + ('>=', '3') + >>> parse_python_version(">=3.8.0") + ('>=', '3.8') + >>> parse_python_version(">=3.8.0.1") + ('>=', '3.8') The version is normalized to "major.minor" (drop patch if present) or "major" if minor is 0 """ # Regex to split operator and version - pattern = r"^(?P\^|~|>=|<=|!=|==|>|<|=)?(?P\d+(\.\d+){0,2})$" + pattern = r"^(?P\^|~|>=|<=|!=|==|>|<|=)?(?P\d+(\.\d+){0,3})$" match = re.match(pattern, selector) if not match: raise ValueError(f"Invalid version selector: {selector}") @@ -321,7 +339,12 @@ def parse_python_version(selector: str): version = match.group("version") # Split into major, minor, and discard the rest (patch or additional parts) - major, minor, *_ = version.split(".") + try: + # Attempt to unpack major, minor, and ignore the rest + major, minor, *_ = version.split(".") + except ValueError: + # If unpacking fails, assume only major is provided + return operator, version # Return only major if minor is "0", otherwise return major.minor return operator, major if minor == "0" else f"{major}.{minor}" diff --git a/tests/test_parse_poetry_version.py b/tests/test_parse_poetry_version.py index 56c8d0a3f..c2c3996ea 100644 --- a/tests/test_parse_poetry_version.py +++ b/tests/test_parse_poetry_version.py @@ -32,6 +32,7 @@ def test_parse_version_failure(invalid_version): ("=3.10", "py==310"), ("==3.10", "py==310"), ("==3", "py==3"), + ("3", "py==3"), (">3.12", "py>312"), ("!=3.7", "py!=37"), # multiple specifiers From 4493140ece12f071f77b07000c20c143c2929129 Mon Sep 17 00:00:00 2001 From: Lorenzo Pirritano Date: Tue, 17 Dec 2024 11:05:29 +0100 Subject: [PATCH 05/10] fix or->and, fix doctests, refact test_combine_conda_selectors --- grayskull/strategy/parse_poetry_version.py | 33 ++++++++++++-------- grayskull/strategy/py_toml.py | 2 +- tests/test_parse_poetry_version.py | 35 +++++++++++----------- tests/test_py_toml.py | 8 ++--- 4 files changed, 43 insertions(+), 35 deletions(-) diff --git a/grayskull/strategy/parse_poetry_version.py b/grayskull/strategy/parse_poetry_version.py index cfff04480..4f931e7c3 100644 --- a/grayskull/strategy/parse_poetry_version.py +++ b/grayskull/strategy/parse_poetry_version.py @@ -257,27 +257,36 @@ def encode_poetry_python_version_to_selector_item(poetry_specifier: str) -> str: """ Encodes Poetry Python version specifier as a Conda selector. - Example: ">=3.8,<3.12" => "py>=38 or py<312" + Example: + ">=3.8,<3.12" => "py>=38 and py<312" + ">=3.8,<3.12,!=3.11" => "py>=38 and py<312 and py!=311" + # TODO: how this case will render? + "<3.8,>=3.10" => "py<38 or py>=310" + "<3.8,>=3.10,!=3.11" => "(py<38 or py>=310) and py!=311" # handle exact version specifiers correctly >>> encode_poetry_python_version_to_selector_item("3") 'py==3' >>> encode_poetry_python_version_to_selector_item("3.8") - "py==38" + 'py==38' >>> encode_poetry_python_version_to_selector_item("==3.8") - "py==38" + 'py==38' >>> encode_poetry_python_version_to_selector_item("!=3.8") - "py!=38" + 'py!=38' + >>> encode_poetry_python_version_to_selector_item("!=3.8.1") + 'py!=38' # handle caret operator correctly - >>> encode_poetry_python_version_to_selector_item("^3.10") - # renders '>=3.10.0,<4.0.0' - "py>=310 or py<4" + >>> encode_poetry_python_version_to_selector_item("^3.10") # '>=3.10.0,<4.0.0' + 'py>=310 and py<4' # handle tilde operator correctly - >>> encode_poetry_python_version_to_selector_item("~3.10") - # renders '>=3.10.0,<3.11.0' - "py>=310 or py<311" + >>> encode_poetry_python_version_to_selector_item("~3.10") # '>=3.10.0,<3.11.0' + 'py>=310 and py<311' + + # handle multiple requirements correctly + >>> encode_poetry_python_version_to_selector_item(">=3.8,<3.12,!=3.11") + 'py>=38 and py<312 and py!=311' """ if not poetry_specifier: @@ -292,7 +301,7 @@ def encode_poetry_python_version_to_selector_item(poetry_specifier: str) -> str: operator, version = parse_python_version(conda_clause) version_selector = version.replace(".", "") conda_selectors.append(f"py{operator}{version_selector}") - selectors = " or ".join(conda_selectors) + selectors = " and ".join(conda_selectors) return selectors @@ -355,8 +364,6 @@ def combine_conda_selectors(python_selector: str, platform_selector: str): Combine selectors based on presence """ if python_selector and platform_selector: - if " or " in python_selector: - python_selector = f"({python_selector})" selector = f"{python_selector} and {platform_selector}" elif python_selector: selector = f"{python_selector}" diff --git a/grayskull/strategy/py_toml.py b/grayskull/strategy/py_toml.py index fd53e817b..3d2437cd4 100644 --- a/grayskull/strategy/py_toml.py +++ b/grayskull/strategy/py_toml.py @@ -44,7 +44,7 @@ def __get_constrained_dep_dict( dep_name: "pandas", result yield: - "pandas >=1.5.0,<2.0.0 # [(py>=38 or py<312) and osx]" + "pandas >=1.5.0,<2.0.0 # [py>=38 and py<312 and osx]" """ conda_version = encode_poetry_version(dep_spec.get("version", "")) if conda_version: diff --git a/tests/test_parse_poetry_version.py b/tests/test_parse_poetry_version.py index c2c3996ea..9ffd11e25 100644 --- a/tests/test_parse_poetry_version.py +++ b/tests/test_parse_poetry_version.py @@ -36,14 +36,17 @@ def test_parse_version_failure(invalid_version): (">3.12", "py>312"), ("!=3.7", "py!=37"), # multiple specifiers - (">3.7,<3.12", "py>37 or py<312"), + (">3.7,<3.12", "py>37 and py<312"), + (">3.7,<3.12,!=3.9", "py>37 and py<312 and py!=39"), + # TODO: how this case will render? + # ("<3.7,>=3.10", "(py<37 or py>=310)"), # poetry specifiers - ("^3.10", "py>=310 or py<4"), - ("~3.10", "py>=310 or py<311"), + ("^3.10", "py>=310 and py<4"), + ("~3.10", "py>=310 and py<311"), # PEP 440 not common specifiers - # ("~=3.7", "<37", "<37"), - # ("3.*", "<37", "<37"), - # ("!=3.*", "<37", "<37"), + # ("~=3.7", "", ""), + # ("3.*", "", ""), + # ("!=3.*", "", ""), ], ) def test_encode_poetry_python_version_to_selector_item( @@ -72,20 +75,18 @@ def test_parse_python_version(python_version, exp_operator_version): @pytest.mark.parametrize( - "python_selector, platform_selector, exp_conda_selector_content", + "python_selector, platform_selector, expected_conda_selector", [ - ("py>=38 or py<312", "osx", "(py>=38 or py<312) and osx"), - ("py>=38 or py<312", "", "py>=38 or py<312"), - ("", "osx", "osx"), - ("py>=38", "", "py>=38"), - ("py<310", "win", "py<310 and win"), + ("", "", ""), + ("py>=38 and py<312", "osx", " # [py>=38 and py<312 and osx]"), + ("py>=38 and py<312", "", " # [py>=38 and py<312]"), + ("", "osx", " # [osx]"), + ("py>=38", "", " # [py>=38]"), + ("py<310", "win", " # [py<310 and win]"), ], ) def test_combine_conda_selectors( - python_selector, platform_selector, exp_conda_selector_content + python_selector, platform_selector, expected_conda_selector ): conda_selector = combine_conda_selectors(python_selector, platform_selector) - expected = ( - f" # [{exp_conda_selector_content}]" if exp_conda_selector_content else "" - ) - assert conda_selector == expected + assert conda_selector == expected_conda_selector diff --git a/tests/test_py_toml.py b/tests/test_py_toml.py index 5e8be26ee..41d36ba1b 100644 --- a/tests/test_py_toml.py +++ b/tests/test_py_toml.py @@ -141,7 +141,7 @@ def test_poetry_get_constrained_dep_caret_version_python_version_min_max_and_pla "pandas", ) ) - == "pandas >=1.5.0,<2.0.0 # [(py>=38 or py<312) and osx]" + == "pandas >=1.5.0,<2.0.0 # [py>=38 and py<312 and osx]" ) @@ -194,7 +194,7 @@ def test_poetry_get_constrained_dep_multiple_constraints_dependencies_with_platf ) == [ "numpy >=1.24.0,<2.0.0 # [py<310]", "numpy >=1.26.0,<2.0.0 # [py>=310]", - "numpy >=1.26.0,<2.0.0 # [(py>=38 or py<310) and osx]", + "numpy >=1.26.0,<2.0.0 # [py>=38 and py<310 and osx]", ] @@ -230,7 +230,7 @@ def test_poetry_get_constrained_dep_multiple_constraints_dependencies_nannyml(): "pandas", ) ) - == "pandas >=1.5.0,<2.0.0 # [py>=38 or py<312]" + == "pandas >=1.5.0,<2.0.0 # [py>=38 and py<312]" ) @@ -244,7 +244,7 @@ def test_poetry_get_constrained_dep_mult_constraints_deps_databricks_sql_connect "pyarrow", ) ) == [ - "pyarrow >=6.0.0 # [py>=37 or py<311]", + "pyarrow >=6.0.0 # [py>=37 and py<311]", "pyarrow >=10.0.1 # [py>=311]", ] From f5fb7aae0df288da2273f6e0268c12b4aa99d4d7 Mon Sep 17 00:00:00 2001 From: Lorenzo Pirritano Date: Tue, 17 Dec 2024 11:05:58 +0100 Subject: [PATCH 06/10] update langchain snapshot test --- tests/data/poetry/langchain-expected.yaml | 6 +++--- tests/test_py_toml.py | 14 ++++++++++++++ 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/tests/data/poetry/langchain-expected.yaml b/tests/data/poetry/langchain-expected.yaml index d2dceafe4..68b2a6a88 100644 --- a/tests/data/poetry/langchain-expected.yaml +++ b/tests/data/poetry/langchain-expected.yaml @@ -44,14 +44,14 @@ requirements: - beautifulsoup4 >=4.0.0,<5.0.0 - pytorch >=1.0.0,<2.0.0 - jinja2 >=3.0.0,<4.0.0 - - tiktoken >=0.0.0,<1.0.0 + - tiktoken >=0.0.0,<1.0.0 # [py>=39 and py<4] - pinecone-client >=2.0.0,<3.0.0 - weaviate-client >=3.0.0,<4.0.0 - google-api-python-client 2.70.0 - wolframalpha 5.0.0 - anthropic >=0.2.2,<0.3.0 - - qdrant-client >=1.0.4,<2.0.0 - - tensorflow-text >=2.11.0,<3.0.0 + - qdrant-client >=1.0.4,<2.0.0 # [py>=38 and py<312] + - tensorflow-text >=2.11.0,<3.0.0 # [py>=310 and py<4 and py<312] - cohere >=3.0.0,<4.0.0 - openai >=0.0.0,<1.0.0 - nlpcloud >=1.0.0,<2.0.0 diff --git a/tests/test_py_toml.py b/tests/test_py_toml.py index 41d36ba1b..9f8b9cd7d 100644 --- a/tests/test_py_toml.py +++ b/tests/test_py_toml.py @@ -78,6 +78,8 @@ def test_poetry_langchain_snapshot(tmpdir): output_path = tmpdir / "langchain" / "meta.yaml" parser = init_parser() + # Check pyproject.toml for version 0.0.119 + # https://inspector.pypi.io/project/langchain/0.0.119 args = parser.parse_args( ["pypi", "langchain==0.0.119", "-o", str(tmpdir), "-m", "AddYourGitHubIdHere"] ) @@ -249,6 +251,18 @@ def test_poetry_get_constrained_dep_mult_constraints_deps_databricks_sql_connect ] +def test_poetry_get_constrained_dep_mult_constraints_deps_langchain_0_0_119(): + assert ( + next( + get_constrained_dep( + {"version": "^2.11.0", "optional": True, "python": "^3.10, <3.12"}, + "tensorflow-text", + ) + ) + == "tensorflow-text >=2.11.0,<3.0.0 # [py>=310 and py<4 and py<312]" + ) + + def test_poetry_entrypoints(): poetry = { "requirements": {"host": ["setuptools"], "run": ["python"]}, From aa72c622fce9237dbc098968a100f9466ce03134 Mon Sep 17 00:00:00 2001 From: Lorenzo Pirritano Date: Tue, 17 Dec 2024 12:49:20 +0100 Subject: [PATCH 07/10] add "or" support --- grayskull/strategy/parse_poetry_version.py | 21 +++++++++++++++++---- tests/test_parse_poetry_version.py | 6 ++++-- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/grayskull/strategy/parse_poetry_version.py b/grayskull/strategy/parse_poetry_version.py index 4f931e7c3..bb7325d8c 100644 --- a/grayskull/strategy/parse_poetry_version.py +++ b/grayskull/strategy/parse_poetry_version.py @@ -260,9 +260,8 @@ def encode_poetry_python_version_to_selector_item(poetry_specifier: str) -> str: Example: ">=3.8,<3.12" => "py>=38 and py<312" ">=3.8,<3.12,!=3.11" => "py>=38 and py<312 and py!=311" - # TODO: how this case will render? - "<3.8,>=3.10" => "py<38 or py>=310" - "<3.8,>=3.10,!=3.11" => "(py<38 or py>=310) and py!=311" + "<3.8|>=3.10" => "py<38 or py>=310" + "<3.8|>=3.10,!=3.11" => "py<38 or py>=310 and py!=311" # handle exact version specifiers correctly >>> encode_poetry_python_version_to_selector_item("3") @@ -284,9 +283,13 @@ def encode_poetry_python_version_to_selector_item(poetry_specifier: str) -> str: >>> encode_poetry_python_version_to_selector_item("~3.10") # '>=3.10.0,<3.11.0' 'py>=310 and py<311' - # handle multiple requirements correctly + # handle multiple requirements correctly (in "and") >>> encode_poetry_python_version_to_selector_item(">=3.8,<3.12,!=3.11") 'py>=38 and py<312 and py!=311' + + # handle multiple requirements in "or" correctly ("and" takes precendence) + >>> encode_poetry_python_version_to_selector_item("<3.8|>=3.10,!=3.11") + 'py<38 or py>=310 and py!=311' """ if not poetry_specifier: @@ -294,6 +297,16 @@ def encode_poetry_python_version_to_selector_item(poetry_specifier: str) -> str: version_specifier = encode_poetry_version(poetry_specifier) + if "|" in version_specifier: + poetry_or_clauses = [clause.strip() for clause in version_specifier.split("|")] + conda_or_clauses = [ + encode_poetry_python_version_to_selector_item(clause) + for clause in poetry_or_clauses + if clause != "" + ] + conda_or_clauses = " or ".join(conda_or_clauses) + return conda_or_clauses + conda_clauses = version_specifier.split(",") conda_selectors = [] diff --git a/tests/test_parse_poetry_version.py b/tests/test_parse_poetry_version.py index 9ffd11e25..eed54011a 100644 --- a/tests/test_parse_poetry_version.py +++ b/tests/test_parse_poetry_version.py @@ -38,8 +38,10 @@ def test_parse_version_failure(invalid_version): # multiple specifiers (">3.7,<3.12", "py>37 and py<312"), (">3.7,<3.12,!=3.9", "py>37 and py<312 and py!=39"), - # TODO: how this case will render? - # ("<3.7,>=3.10", "(py<37 or py>=310)"), + # version specifiers in "or" ("and" takes precedence) + ("<3.7|>=3.10", "py<37 or py>=310"), + ("<3.8|>=3.10,!=3.11", "py<38 or py>=310 and py!=311"), + ("<3.8|>=3.10|=3.9", "py<38 or py>=310 or py==39"), # poetry specifiers ("^3.10", "py>=310 and py<4"), ("~3.10", "py>=310 and py<311"), From 5abdffd49ea3b81ef6fad5716958766a9cd1c471 Mon Sep 17 00:00:00 2001 From: Lorenzo Pirritano Date: Tue, 17 Dec 2024 14:00:12 +0100 Subject: [PATCH 08/10] wrap again python selector if there is an "or" and platform specified --- grayskull/strategy/parse_poetry_version.py | 2 ++ tests/test_parse_poetry_version.py | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/grayskull/strategy/parse_poetry_version.py b/grayskull/strategy/parse_poetry_version.py index bb7325d8c..8ad1e17d8 100644 --- a/grayskull/strategy/parse_poetry_version.py +++ b/grayskull/strategy/parse_poetry_version.py @@ -377,6 +377,8 @@ def combine_conda_selectors(python_selector: str, platform_selector: str): Combine selectors based on presence """ if python_selector and platform_selector: + if " or " in python_selector: + python_selector = f"({python_selector})" selector = f"{python_selector} and {platform_selector}" elif python_selector: selector = f"{python_selector}" diff --git a/tests/test_parse_poetry_version.py b/tests/test_parse_poetry_version.py index eed54011a..cbf940b5a 100644 --- a/tests/test_parse_poetry_version.py +++ b/tests/test_parse_poetry_version.py @@ -85,6 +85,12 @@ def test_parse_python_version(python_version, exp_operator_version): ("", "osx", " # [osx]"), ("py>=38", "", " # [py>=38]"), ("py<310", "win", " # [py<310 and win]"), + ( + "py<38 or py>=310 and py!=311", + "osx", + " # [(py<38 or py>=310 and py!=311) and osx]", + ), + ("py<38 or py>=310", "osx", " # [(py<38 or py>=310) and osx]"), ], ) def test_combine_conda_selectors( From 602c447e61decde78dbd67e40a473d80a0a03eb3 Mon Sep 17 00:00:00 2001 From: Lorenzo Pirritano Date: Tue, 17 Dec 2024 14:29:08 +0100 Subject: [PATCH 09/10] add one test for python version in "or" --- tests/test_py_toml.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/test_py_toml.py b/tests/test_py_toml.py index 9f8b9cd7d..a6ac1fa0f 100644 --- a/tests/test_py_toml.py +++ b/tests/test_py_toml.py @@ -147,6 +147,21 @@ def test_poetry_get_constrained_dep_caret_version_python_version_min_max_and_pla ) +def test_poetry_get_constrained_dep_caret_version_python_version_in_or_and_platform(): + assert next( + get_constrained_dep( + { + "version": "^1.5", + "python": "<=3.7,!=3.4|>=3.10,!=3.12", + "platform": "darwin", + }, + "pandas", + ) + ) == ( + "pandas >=1.5.0,<2.0.0 # [(py<=37 and py!=34 or py>=310 and py!=312) and osx]" + ) + + def test_poetry_get_constrained_dep_no_version_only_platform(): assert ( next( From ee3a6d6a85fdcb0d786360bb2f26b776d54ee34a Mon Sep 17 00:00:00 2001 From: Lorenzo Pirritano Date: Sun, 22 Dec 2024 16:12:08 +0100 Subject: [PATCH 10/10] add "~=" and ".*" support --- grayskull/strategy/parse_poetry_version.py | 425 ++++++++++++++++++--- tests/test_parse_poetry_version.py | 49 ++- tests/test_py_toml.py | 26 ++ 3 files changed, 441 insertions(+), 59 deletions(-) diff --git a/grayskull/strategy/parse_poetry_version.py b/grayskull/strategy/parse_poetry_version.py index 8ad1e17d8..d702961f5 100644 --- a/grayskull/strategy/parse_poetry_version.py +++ b/grayskull/strategy/parse_poetry_version.py @@ -1,6 +1,7 @@ import re import semver +from packaging.version import Version VERSION_REGEX = re.compile( r"""^[vV]? @@ -149,6 +150,8 @@ def encode_poetry_version(poetry_specifier: str) -> str: Example: ^1 => >=1.0.0,<2.0.0 # should be unchanged + >>> encode_poetry_version("~=1.1") + '~=1.1' >>> encode_poetry_version("1.*") '1.*' >>> encode_poetry_version(">=1,<2") @@ -157,6 +160,8 @@ def encode_poetry_version(poetry_specifier: str) -> str: '==1.2.3' >>> encode_poetry_version("!=1.2.3") '!=1.2.3' + >>> encode_poetry_version("===1.2.3") + '===1.2.3' # strip spaces >>> encode_poetry_version(">= 1, < 2") @@ -223,6 +228,12 @@ def encode_poetry_version(poetry_specifier: str) -> str: conda_clauses.append("<" + ceiling) continue + if poetry_clause.startswith("~="): + # handle the compatible release operator ~= + # before the tilde ~ operator + conda_clauses.append(poetry_clause) + continue + if poetry_clause.startswith("~"): # handle ~ operator target = poetry_clause[1:] @@ -255,7 +266,7 @@ def encode_poetry_platform_to_selector_item(poetry_platform: str) -> str: def encode_poetry_python_version_to_selector_item(poetry_specifier: str) -> str: """ - Encodes Poetry Python version specifier as a Conda selector. + Encodes Poetry Python version specifier set as a Conda selector. Example: ">=3.8,<3.12" => "py>=38 and py<312" @@ -290,6 +301,29 @@ def encode_poetry_python_version_to_selector_item(poetry_specifier: str) -> str: # handle multiple requirements in "or" correctly ("and" takes precendence) >>> encode_poetry_python_version_to_selector_item("<3.8|>=3.10,!=3.11") 'py<38 or py>=310 and py!=311' + + # handle compatible release operator correctly + >>> encode_poetry_python_version_to_selector_item("~=3") + 'py>=3' + >>> encode_poetry_python_version_to_selector_item("~=3.8") + 'py>=38 and py<4' + >>> encode_poetry_python_version_to_selector_item("~=3.8.1") + 'py==38' + >>> encode_poetry_python_version_to_selector_item("~=3.8.0.1") + 'py==38' + >>> encode_poetry_python_version_to_selector_item("~=3.8,!=3.11") + 'py>=38 and py<4 and py!=311' + + # handle wildcard versions correctly + >>> encode_poetry_python_version_to_selector_item("*") + '' + >>> encode_poetry_python_version_to_selector_item("3.*,!=3.11") + 'py>=3 and py<4 and py!=311' + >>> encode_poetry_python_version_to_selector_item("!=3.*|3.11") + 'py<3 or py>=4 or py==311' + >>> encode_poetry_python_version_to_selector_item("!=3.*,!=4.1") + '(py<3 or py>=4) and py!=41' + """ if not poetry_specifier: @@ -311,65 +345,368 @@ def encode_poetry_python_version_to_selector_item(poetry_specifier: str) -> str: conda_selectors = [] for conda_clause in conda_clauses: - operator, version = parse_python_version(conda_clause) - version_selector = version.replace(".", "") - conda_selectors.append(f"py{operator}{version_selector}") + conda_selector = parse_python_version_specifier_to_selector(conda_clause) + if conda_selector != "": + conda_selectors.append(conda_selector) + if len(conda_selectors) > 1: + conda_selectors = [ + conda_selector if " or " not in conda_selector else f"({conda_selector})" + for conda_selector in conda_selectors + ] selectors = " and ".join(conda_selectors) return selectors -def parse_python_version(selector: str): +def parse_python_version_specifier_to_selector(version_specifier: str): """ - Return operator and normalized version from a version selector + Take a Python version specifier, PEP 440 compliant. - Examples: - ">=3" -> ">=", "3" - ">=3.0" -> ">=", "3" - ">=3.8" -> ">=", "3.8" - ">=3.8.0" -> ">=", "3.8" - "<4.0.0" -> "<", "4" - "3.12" -> "==", 3.12" - "=3.8" -> "==", "3.8" - ">=3.8.0.1" -> ">=", "3.8" - - >>> parse_python_version(">=3.8") - ('>=', '3.8') - >>> parse_python_version("3.12") - ('==', '3.12') - >>> parse_python_version("<4.0.0") - ('<', '4') - >>> parse_python_version(">=3") - ('>=', '3') - >>> parse_python_version(">=3.8.0") - ('>=', '3.8') - >>> parse_python_version(">=3.8.0.1") - ('>=', '3.8') + Return Python version conda selector. + + If the version_specifier has no operator, the equal operator == + is assumed. The version is normalized to "major.minor" (drop patch if present) - or "major" if minor is 0 + or only "major" if minor is 0 (e.g. "3.8" -> "38", "3.8.1" -> "38", + "3.0" -> "3", "3.0.1" -> "3"). + + The compatible release operator ~= is expanded eventually into two selector + items if the version has major and minor (e.g. "~=3.8" -> "py>=38 and py<4", + but "~=3.8.1" -> "py==38"). + + The exact equality operators == and != support the wildcard * + in the version (e.g. "*" -> "==*" -> ""). + + Examples: + ">=3.8" -> "py>=38" + "3.12" -> "py==312" + "~=3.8" -> "py>=38 and py<4" + "~=3.8.1" -> "py==38" + "3.*" -> "py>=3 and py<4" + "!=3.*" -> "py<3 or py>=4" + + >>> parse_python_version_specifier_to_selector(">=3.8") + 'py>=38' + >>> parse_python_version_specifier_to_selector("3.12") + 'py==312' + >>> parse_python_version_specifier_to_selector("<4.0.0") + 'py<4' + >>> parse_python_version_specifier_to_selector("<4.0.0.1") + 'py<4' + >>> parse_python_version_specifier_to_selector(">=3") + 'py>=3' + >>> parse_python_version_specifier_to_selector(">=3.8.0") + 'py>=38' + >>> parse_python_version_specifier_to_selector(">=3.8.0.1") + 'py>=38' + >>> parse_python_version_specifier_to_selector("~=3.8") + 'py>=38 and py<4' + >>> parse_python_version_specifier_to_selector("3.*") + 'py>=3 and py<4' + >>> parse_python_version_specifier_to_selector("!=3.*") + 'py<3 or py>=4' + """ - # Regex to split operator and version - pattern = r"^(?P\^|~|>=|<=|!=|==|>|<|=)?(?P\d+(\.\d+){0,3})$" - match = re.match(pattern, selector) + # Regex to split an optional operator and a whatever version + pattern = r"^(?P\^|~=|~|>=|<=|>|<|!=|===|==|=)?(?P.+)$" + + # Here Specifier or Version are not useful because + # Specifier requires an operator, and Version cannot + # accept an operator. Doomed to match twice. + + match = re.match(pattern, version_specifier) if not match: - raise ValueError(f"Invalid version selector: {selector}") + raise ValueError(f"Invalid version selector: {version_specifier}") # Extract operator and version operator = match.group("operator") - # Default to "==" if no operator is provided or "=" - operator = "==" if operator in {None, "="} else operator version = match.group("version") - # Split into major, minor, and discard the rest (patch or additional parts) - try: - # Attempt to unpack major, minor, and ignore the rest - major, minor, *_ = version.split(".") - except ValueError: - # If unpacking fails, assume only major is provided - return operator, version + if operator in [None, "=", "==", "==="]: + # Default to "==" if no operator is provided or "=", "===" + operator = "==" + # Check also if there is a wildcard operator "*" (may result in two operators) + return expand_operator_wildcard_version_to_selector(operator, version) + elif operator == "~=": + # Compatible release operator "~=" (may result in two operators) + return expand_compatible_release_operator_version_to_selector(version) + elif operator == "!=": + # Check also if there is a wildcard operator "*" (may result in two operators) + return expand_operator_wildcard_version_to_selector(operator, version) + return operator_version_to_selector(operator, Version(version)) + + +def expand_compatible_release_operator_version_to_selector( + version: str | Version, +) -> str: + """ + Take a Python version, PEP440 compliant. + + The compatible release operator "~=" is implicit. + + The python version should be reasonable and realistic (e.g. "3.11"), but + it is true that an esoteric still PEP440 valid version would make grayskull + crash, therefore here we parse the Python version as a PEP440 compliant + version (e.g. "3.11.3.dev0"). + + The compatible release operator ~= is expanded eventually into two selector + items if the version has major and minor (e.g. "~=3.8" -> "py>=38 and py<4", + but "~=3.8.1" -> "py==38", and "~=3" -> "py>=38"). + + The reason why this operator is expanded here and not in + encode_poetry_python_version_to_selector_item just after + encode_poetry_version is because the selectors for the python + version use only major and minor, and therefore the compatible + release operator ~= makes sense only in the case of major and + minor specified (e.g. "~=3.8" -> "py>=38 and py<4") and just by + knowing that we can avoid having to detect cases like: + "~=3.8.0.1" -> ">=3.8.0.1, ==3.8.0.*" -> ">=3.8.0.1, <3.8.1.0a" -> "py==38" + and expand only for: + "~=3.8" -> ">=3.8, ==3.*" -> ">=3.8, <4.0a" -> "py>=38 and py<4" + "~=3.0" -> ">=3.0, ==3.*" -> ">=3.0, <4.0a" -> "py>=3 and py<4" + in the rest of the cases it's a simple conversion to ">=" operator. + + If we would expand it before, we would receive specifier sets like + ">=3.8.0.1, <3.8.1.0a" (among other specifiers) and we would need to + detect those cases to avoid rendering to a naive "py>=38 and py<38" + which would be an invalid statement. + + Rationale: + - generally it would work in this way: + ~=2 -> illegal for PEP440 + ~=2.2 -> ">=2.2, ==2.*" -> ">=2.2, <3.0a" + ~=1.4.5 -> ">=1.4.5, ==1.4.*" -> ">=1.4.5, <1.5.0a" + ~=0.5.3 -> ">=0.5.3, ==0.5.*" -> ">=0.5.3, <0.6.0a" + - considering only python versions and their selectors: + ~=3 -> illegal for PEP440 -> ">=3, ==*" -> ">=3" -> "py>=3" + ~=3.8 -> ">=3.8, ==3.*" -> ">=3.8, <4.0a" -> "py>=38 and py<4" + ~=3.8.1 -> ">=3.8.1, ==3.8.*" -> ">=3.8.1, <3.9.0a" -> "py==38" + ~=3.8.0.1 -> ">=3.8.0.1, ==3.8.0.*" -> ">=3.8.0.1, <3.8.1.0a" -> "py==38" + + Examples: + "3" -> "py>=3" + "3.8" -> "py>=38 and py<4" + "3.8.1" -> "py==38" + + >>> expand_compatible_release_operator_version_to_selector("3") + 'py>=3' + >>> expand_compatible_release_operator_version_to_selector("3.8") + 'py>=38 and py<4' + >>> expand_compatible_release_operator_version_to_selector("3.0") + 'py>=3 and py<4' + >>> expand_compatible_release_operator_version_to_selector("3.8.1") + 'py==38' + >>> expand_compatible_release_operator_version_to_selector("3.8.1.1") + 'py==38' + >>> expand_compatible_release_operator_version_to_selector("3.8a0") + 'py>=38 and py<4' + >>> expand_compatible_release_operator_version_to_selector("3.8.1.post1") + 'py==38' + """ + if not isinstance(version, Version): + version = Version(version) + + # The compatible release operator ~= is expanded eventually + # into two selector items if the version has major and minor + # even if the minor is 0, because it would be used as a padding + # placeholder. + # See: + # https://packaging.python.org/en/latest/specifications/version-specifiers/#compatible-release + if len(version.release) < 3: + # "3.8" -> "py>=38 and py<4", and "3" -> "py>=3" + lower_bound_operator = ">=" + else: + # "3.8.1" -> "py==38" + lower_bound_operator = "==" + lower_bound_selector = operator_version_to_selector(lower_bound_operator, version) + if len(version.release) == 2: + # get version selector, with ">=" operator as lower bound + # get the ceiling of the version (major bumped by 1) + ceiling_version = version.major + 1 + # get ceiling version selector, with "<" operator as upper bound + upper_bound = operator_version_to_selector("<", Version(f"{ceiling_version}")) + return f"{lower_bound_selector} and {upper_bound}" + return lower_bound_selector + + +def expand_operator_wildcard_version_to_selector( + operator: str | None, version: str | Version +) -> str: + """ + Take the strict equality operators "==" or "!=" and + a Python version ending with ".*", PEP440 compliant. + + "*" is accepted, but "1*" or "1.1*" are not accepted + because PEP 440 requires the "*" wildcard to follow a "." + because it is meant to represent a "range of versions + with common prefix components." - # Return only major if minor is "0", otherwise return major.minor - return operator, major if minor == "0" else f"{major}.{minor}" + Wildcards can be expressed as ranges (">=" and "<") using + the next significant component. + + Examples: + "*" -> "==*" -> "" + "1.*" -> ">=1.0.0.a0,<2.0.0" + "1.1.*" -> ">=1.1.0.a0,<1.2.0" + "1.1.1.*" -> ">=1.1.1.a0,<1.1.2" + + inclusion: 1.1 + == 1.1 # Equal, so 1.1 matches clause + == 1.1.0 # Zero padding expands 1.1 to 1.1.0, so it matches clause + == 1.1.dev1 # Not equal (dev-release), so 1.1 does not match clause + == 1.1a1 # Not equal (pre-release), so 1.1 does not match clause + == 1.1.post1 # Not equal (post-release), so 1.1 does not match clause + == 1.1.* # Same prefix, so 1.1 matches clause + + exclusion: 1.1.post1 + != 1.1 # Not equal, so 1.1.post1 matches clause + != 1.1.post1 # Equal, so 1.1.post1 does not match clause + != 1.1.* # Same prefix, so 1.1.post1 does not match clause + + # In practice, if the star suffix ".*" is used on Python version specifiers + # to be rendered as conda selector, we can simplify the calculation according + # to which operator is used: + # + # - equality: it makes sense to expand it only if the version is "{major}.*" + # (e.g. "3.*" -> ">=3.0a0,<4" -> "py>=3 and py<4"). In all the + # other cases it is enough to remove the ".*" and consider as + # usual the "{operator}{major}{minor}" if "minor" is more than + # "0", otherwise "{operator}{major}". + + # - inequality: it makes sense to expand it only if the version is "{major}.*" + # (e.g. "3.*" -> ">=3.0a0,<4" -> "py>=3 and py<4"). In all the + # other cases it is enough to remove the ".*" and consider as + # usual the "{operator}{major}{minor}" if "minor" is more than + # "0", otherwise "{operator}{major}". + + # Equality examples + + >>> expand_operator_wildcard_version_to_selector("==","*") # any + '' + + >>> expand_operator_wildcard_version_to_selector("==", "3.*") # >=3.0a0,<4 + 'py>=3 and py<4' + + # >=3.12.0a0,<3.13 + >>> expand_operator_wildcard_version_to_selector("==", "3.12.*") + 'py==312' + + # >=3.9.1.0a0,<3.9.2 + >>> expand_operator_wildcard_version_to_selector("==", "3.9.1.*") + 'py==39' + + # >=3.9.1.0a0,<3.9.1.2 + >>> expand_operator_wildcard_version_to_selector("==", "3.9.1.1.*") + 'py==39' + + # Inequality examples + + >>> expand_operator_wildcard_version_to_selector("!=","*") # none + 'py<0' + + # <3.0a0|>=4 + >>> expand_operator_wildcard_version_to_selector("!=", "3.*") + 'py<3 or py>=4' + + # <3.12.0a0|>=3.13 + >>> expand_operator_wildcard_version_to_selector("!=", "3.12.*") + 'py<312 or py>=313' + + # <3.9.1.0a0|>=3.9.2 + >>> expand_operator_wildcard_version_to_selector("!=", "3.9.1.*") + 'py<39 or py>=39' + + # <3.9.1.1.0a0|>=3.9.1.2 + >>> expand_operator_wildcard_version_to_selector("!=", "3.9.1.1.*") + 'py<39 or py>=39' + + """ + if version == "*": + # This should not happen, as the "*" is stripped away before + # to avoid having trivial selectors, but consider it anyway + # for general usage. + if operator in [None, "", "=", "==", "==="]: + return "" + else: + return "py<0" + base_version = version.rstrip(".*") + expand_operator_wildcard_version = len(base_version) != len(version) + version = Version(base_version) + if operator in [None, "", "=", "==", "==="]: + # Default to "==" if no operator is provided or "=", "===" + operator = "==" + # it makes sense to expand it only if the version is "{major}.*" + if expand_operator_wildcard_version and len(version.release) == 1: + # "3.*" -> ">=3.0a0,<4" -> "py>=3 and py<4" + left_bound_selector = operator_version_to_selector(">=", version) + # get the ceiling of the version (major bumped by 1) + right_version = version.major + 1 + # get ceiling version selector, with "<" operator as upper bound + right_bound_selector = operator_version_to_selector( + "<", Version(f"{right_version}") + ) + return f"{left_bound_selector} and {right_bound_selector}" + elif operator == "!=": + if expand_operator_wildcard_version: + left_bound_selector = operator_version_to_selector("<", version) + if len(version.release) == 1: + # "3.*" -> "<3.0a0|>=4" -> "py<3 or py>=4" + # major bumped by 1 + right_version = version.major + 1 + elif len(version.release) == 2: + # "3.12.*" -> "<3.12.0a0|>=3.13" -> "py<312 or py>=313" + # minor bumped by 1 + right_version = f"{version.major}." + str(version.minor + 1) + else: + # "3.9.1.*" -> <3.9.1.0a0|>=3.9.2"" -> "py<39 or py>=39" + # "3.9.1.1.*" -> <3.9.1.1.0a0|>=3.9.1.2"" -> "py<39 or py>=39" + # use the same version in the right bound + right_version = f"{version.major}.{version.minor}" + # get ceiling version selector, with ">=" operator as upper bound + right_bound_selector = operator_version_to_selector( + ">=", Version(f"{right_version}") + ) + return f"{left_bound_selector} or {right_bound_selector}" + return operator_version_to_selector(operator, version) + + +def operator_version_to_selector(operator: str | None, version: str | Version) -> str: + """ + Consider major, minor, and discard the rest (patch or additional parts) + Return only major if minor is "0", otherwise return major.minor + + >>> operator_version_to_selector(">=", "3.8") + 'py>=38' + >>> operator_version_to_selector("==", "3.12") + 'py==312' + >>> operator_version_to_selector("", "3.12") + 'py==312' + >>> operator_version_to_selector("<", "4.0.0") + 'py<4' + >>> operator_version_to_selector("<", "4.0.0.1") + 'py<4' + >>> operator_version_to_selector(">=", "3") + 'py>=3' + >>> operator_version_to_selector(">=", "3.8.0") + 'py>=38' + >>> operator_version_to_selector(">=", "3.8.0.1") + 'py>=38' + >>> operator_version_to_selector(">=", "3.8.0.1post1") + 'py>=38' + >>> operator_version_to_selector(">=", "3.8.0.1a0") + 'py>=38' + >>> operator_version_to_selector("<", "2!4.0.0.1.post1") + 'py<4' + """ + if operator in [None, "", "=", "==="]: + # Default to "==" if no operator is provided or "=", "===" + operator = "==" + if not isinstance(version, Version): + version = Version(version) + version_selector = ( + version.major if version.minor == 0 else f"{version.major}{version.minor}" + ) + return f"py{operator}{version_selector}" def combine_conda_selectors(python_selector: str, platform_selector: str): diff --git a/tests/test_parse_poetry_version.py b/tests/test_parse_poetry_version.py index cbf940b5a..c25d3529a 100644 --- a/tests/test_parse_poetry_version.py +++ b/tests/test_parse_poetry_version.py @@ -6,7 +6,7 @@ InvalidVersion, combine_conda_selectors, encode_poetry_python_version_to_selector_item, - parse_python_version, + parse_python_version_specifier_to_selector, parse_version, ) @@ -46,9 +46,9 @@ def test_parse_version_failure(invalid_version): ("^3.10", "py>=310 and py<4"), ("~3.10", "py>=310 and py<311"), # PEP 440 not common specifiers - # ("~=3.7", "", ""), - # ("3.*", "", ""), - # ("!=3.*", "", ""), + ("~=3.8", "py>=38 and py<4"), + ("3.*", "py>=3 and py<4"), + ("!=3.*", "py<3 or py>=4"), ], ) def test_encode_poetry_python_version_to_selector_item( @@ -60,20 +60,39 @@ def test_encode_poetry_python_version_to_selector_item( @pytest.mark.parametrize( - "python_version, exp_operator_version", + "python_version, expected_conda_selector", [ - (">=3.8", (">=", "3.8")), - (">=3.8.0", (">=", "3.8")), - ("<4.0.0", ("<", "4")), - ("3.12", ("==", "3.12")), - ("=3.8", ("==", "3.8")), - ("=3.8.1", ("==", "3.8")), - ("3.8.1", ("==", "3.8")), + (">=3", "py>=3"), + (">=3.8", "py>=38"), + (">=3.8.0", "py>=38"), + (">=3.8.0.1", "py>=38"), + ("<4.0.0.0", "py<4"), + ("<4.0.0", "py<4"), + ("<4.0", "py<4"), + ("3", "py==3"), + ("3.12", "py==312"), + ("3.12.1", "py==312"), + ("3.12.1.1", "py==312"), + ("=3", "py==3"), + ("=3.8", "py==38"), + ("=3.8.1", "py==38"), + ("=3.8.1.1", "py==38"), + ("===3", "py==3"), + ("===3.8", "py==38"), + ("===3.8.1", "py==38"), + ("===3.8.1.1", "py==38"), + ("!=3", "py!=3"), + ("!=3.8", "py!=38"), + ("!=3.8.1", "py!=38"), + ("!=3.8.1.1", "py!=38"), + ("~=3.8", "py>=38 and py<4"), ], ) -def test_parse_python_version(python_version, exp_operator_version): - operator, version = parse_python_version(python_version) - assert (operator, version) == exp_operator_version +def test_parse_python_version_specifier_to_selector( + python_version, expected_conda_selector +): + conda_selector = parse_python_version_specifier_to_selector(python_version) + assert conda_selector == expected_conda_selector @pytest.mark.parametrize( diff --git a/tests/test_py_toml.py b/tests/test_py_toml.py index a6ac1fa0f..953f654c6 100644 --- a/tests/test_py_toml.py +++ b/tests/test_py_toml.py @@ -162,6 +162,32 @@ def test_poetry_get_constrained_dep_caret_version_python_version_in_or_and_platf ) +def test_poetry_get_constrained_dep_compatible_rel_op_python_version_and_platform(): + assert next( + get_constrained_dep( + { + "version": "^1.5", + "python": "~=3.8", + "platform": "darwin", + }, + "pandas", + ) + ) == ("pandas >=1.5.0,<2.0.0 # [py>=38 and py<4 and osx]") + + +def test_poetry_get_constrained_dep_wildvard_python_version_and_platform(): + assert next( + get_constrained_dep( + { + "version": "^1.5", + "python": "3.*", + "platform": "darwin", + }, + "pandas", + ) + ) == ("pandas >=1.5.0,<2.0.0 # [py>=3 and py<4 and osx]") + + def test_poetry_get_constrained_dep_no_version_only_platform(): assert ( next(