Skip to content

Commit

Permalink
Add several pypi tests
Browse files Browse the repository at this point in the history
These tests are designed to make sure multiple selectors for the
same package are preserved correctly at every stage of the process
of merging dependencies from different sources.

When parsing poetry dependencies, we introduce selectors early
in the process, whereas for other approaches, we retain "extras"
(separated by semicolons) for much of the merging process. Packages
employing each strategy are used in the added tests.
  • Loading branch information
xylar committed Dec 29, 2024
1 parent 62c1fee commit 3c85944
Showing 1 changed file with 168 additions and 0 deletions.
168 changes: 168 additions & 0 deletions tests/test_pypi.py
Original file line number Diff line number Diff line change
Expand Up @@ -682,6 +682,124 @@ def test_merge_pypi_sdist_metadata():
)


def test_pypi_metadata_constraints_for_python_versions():
config = Configuration(name="apache-airflow-providers-trino",
version="6.0.0")
pypi_metadata = get_pypi_metadata(config)
assert sorted(pypi_metadata["requires_dist"]) == sorted(
[
'apache-airflow-providers-common-sql>=1.20.0',
'apache-airflow>=2.9.0',
'pandas<2.2,>=1.5.3; python_version < "3.9"',
'pandas<2.2,>=2.1.2; python_version >= "3.9"',
'trino>=0.318.0',
'apache-airflow-providers-google; extra == "google"',
'apache-airflow-providers-openlineage; extra == "openlineage"'
]
)

config = Configuration(name="databricks-sql-connector", version="3.7.0")
pypi_metadata = get_pypi_metadata(config)
assert sorted(pypi_metadata["requires_dist"]) == sorted(
[
'alembic<2.0.0,>=1.0.11; extra == "alembic"',
'lz4<5.0.0,>=4.0.2',
'numpy>=1.16.6; python_version >= "3.8" and python_version < "3.11"',
'numpy>=1.23.4; python_version >= "3.11"',
'oauthlib<4.0.0,>=3.1.0',
'openpyxl<4.0.0,>=3.0.10',
'pandas<2.3.0,>=1.2.5; python_version >= "3.8"',
'pyarrow>=14.0.1',
'requests<3.0.0,>=2.18.1',
'sqlalchemy>=2.0.21; extra == "sqlalchemy" or extra == "alembic"',
'thrift<0.21.0,>=0.16.0',
'urllib3>=1.26',
]
)


def test_sdist_metadata_from_toml_project_dependencies():
config = Configuration(name="apache-airflow-providers-trino",
version="6.0.0")
pypi_metadata = get_pypi_metadata(config)
sdist_metadata = get_sdist_metadata(pypi_metadata["sdist_url"], config)
assert sorted(sdist_metadata["install_requires"]) == sorted(
[
'apache-airflow-providers-common-sql>=1.20.0',
'apache-airflow>=2.9.0',
'pandas>=1.5.3,<2.2;python_version<"3.9"',
'pandas>=2.1.2,<2.2;python_version>="3.9"',
'trino>=0.318.0',
'python ~=3.9'
]
)


def test_sdist_metadata_from_toml_poetry_dependencies():
config = Configuration(name="databricks-sql-connector", version="3.7.0")
pypi_metadata = get_pypi_metadata(config)
sdist_metadata = get_sdist_metadata(pypi_metadata["sdist_url"], config)
assert sorted(sdist_metadata["install_requires"]) == sorted(
[
'python >=3.8.0,<4.0.0',
'thrift >=0.16.0,<0.21.0',
'pandas >=1.2.5,<2.3.0 # [py>=38]',
'pyarrow >=14.0.1',
'lz4 >=4.0.2,<5.0.0',
'requests >=2.18.1,<3.0.0',
'oauthlib >=3.1.0,<4.0.0',
'numpy >=1.16.6 # [py>=38 and py<311]',
'numpy >=1.23.4 # [py>=311]',
'openpyxl >=3.0.10,<4.0.0',
'urllib3 >=1.26'
]
)


def test_merge_pypi_sdist_metadata_from_toml():
# tests merging pyproject.toml dependencies from poetry with pypi data,
# including multiple numpy constraints with python version selectors
config = Configuration(name="databricks-sql-connector", version="3.7.0")
pypi_metadata = get_pypi_metadata(config)
sdist_metadata = get_sdist_metadata(pypi_metadata["sdist_url"], config)
merged_data = merge_pypi_sdist_metadata(pypi_metadata, sdist_metadata,
config)
assert sorted(merged_data["requires_dist"]) == sorted(
[
'python >=3.8.0,<4.0.0',
'thrift >=0.16.0,<0.21.0',
'pandas >=1.2.5,<2.3.0 # [py>=38]',
'pyarrow >=14.0.1',
'lz4 >=4.0.2,<5.0.0',
'requests >=2.18.1,<3.0.0',
'oauthlib >=3.1.0,<4.0.0',
'numpy >=1.16.6 # [py>=38 and py<311]',
'numpy >=1.23.4 # [py>=311]',
'openpyxl >=3.0.10,<4.0.0',
'urllib3 >=1.26'
]
)

# tests merging pyproject.toml project dependencies with pypi data,
# including multiple pandas constraints with python version selectors
config = Configuration(name="apache-airflow-providers-trino",
version="6.0.0")
pypi_metadata = get_pypi_metadata(config)
sdist_metadata = get_sdist_metadata(pypi_metadata["sdist_url"], config)
merged_data = merge_pypi_sdist_metadata(pypi_metadata, sdist_metadata,
config)
assert sorted(merged_data["requires_dist"]) == sorted(
[
'apache-airflow-providers-common-sql>=1.20.0',
'apache-airflow>=2.9.0',
'pandas>=1.5.3,<2.2;python_version<"3.9"',
'pandas>=2.1.2,<2.2;python_version>="3.9"',
'trino>=0.318.0',
'python ~=3.9'
]
)


def test_update_requirements_with_pin():
req = {
"build": ["<{ compiler('c') }}"],
Expand Down Expand Up @@ -787,6 +905,56 @@ def test_run_requirements_sdist():
]
)

# a more complex example with selectors
config = Configuration(name="apache-airflow-providers-trino",
version="6.0.0")
recipe = GrayskullFactory.create_recipe("pypi", config)
assert "noarch" not in recipe["build"]
selectors = {
"python >=3.9,<4.dev0": None,
"apache-airflow-providers-common-sql >=1.20.0": None,
"apache-airflow >=2.9.0": None,
"pandas >=1.5.3,<2.2": "[py<39]",
"pandas >=2.1.2,<2.2": "[py>=39]",
"trino-python-client >=0.318.0": None
}

for dep in recipe["requirements"]["run"]:
dep_str = str(dep)
assert dep_str in selectors
selector = selectors[dep_str]
if dep.inline_comment is None:
assert selector is None
else:
assert str(dep.inline_comment) == selector

# another example with selectors
config = Configuration(name="databricks-sql-connector", version="3.7.0")
recipe = GrayskullFactory.create_recipe("pypi", config)
assert "noarch" not in recipe["build"]
selectors = {
"python >=3.8.0,<4.0.0": None,
"thrift >=0.16.0,<0.21.0": None,
"pandas >=1.2.5,<2.3.0": "[py>=38]",
"pyarrow >=14.0.1": None,
"lz4 >=4.0.2,<5.0.0": None,
"requests >=2.18.1,<3.0.0": None,
"oauthlib >=3.1.0,<4.0.0": None,
"numpy >=1.16.6": "[py>=38 and py<311]",
"numpy >=1.23.4": "[py>=311]",
"openpyxl >=3.0.10,<4.0.0": None,
"urllib3 >=1.26": None
}

for dep in recipe["requirements"]["run"]:
dep_str = str(dep)
assert dep_str in selectors
selector = selectors[dep_str]
if dep.inline_comment is None:
assert selector is None
else:
assert str(dep.inline_comment) == selector


def test_format_host_requirements():
assert sorted(format_dependencies(["setuptools>=40.0", "pkg2"], "pkg1")) == sorted(
Expand Down

0 comments on commit 3c85944

Please sign in to comment.