Skip to content

Commit 7c99106

Browse files
TG1999TG1999
committed
Ignore already resolved dependencies
Signed-off-by: Tushar Goel <tushar.goel.dav@gmail.com>
1 parent 1047a8a commit 7c99106

5 files changed

Lines changed: 144 additions & 103 deletions

File tree

‎src/_packagedcode/pypi.py‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -745,7 +745,8 @@ def parse_reqs(cls, reqs, scope):
745745
for req in reqs:
746746
is_resolved = False
747747
req_parsed = packaging.requirements.Requirement(str(req))
748-
purl = PackageURL(type="pypi", name=req_parsed.name)
748+
name = canonicalize_name(req_parsed.name)
749+
purl = PackageURL(type="pypi", name=name)
749750
specifiers = req_parsed.specifier._specs
750751
if len(specifiers) == 1:
751752
specifier = list(specifiers)[0]
@@ -754,7 +755,7 @@ def parse_reqs(cls, reqs, scope):
754755
purl = purl._replace(version=specifier.version)
755756
dependent_packages.append(
756757
models.DependentPackage(
757-
purl=purl,
758+
purl=str(purl),
758759
scope=scope,
759760
is_runtime=True,
760761
is_optional=False,

‎src/python_inspector/resolution.py‎

Lines changed: 119 additions & 93 deletions
Original file line numberDiff line numberDiff line change
@@ -70,12 +70,13 @@ def get_python_version_from_env_tag(python_version: str):
7070

7171

7272
class PythonInputProvider(AbstractProvider):
73-
def __init__(self, environment=None, repos=tuple()):
73+
def __init__(self, environment=None, repos=tuple(), resolved_requirements=[]):
7474
self.environment = environment
7575
self.repos = repos or []
7676
self.versions_by_package = {}
7777
self.dependencies_by_purl = {}
7878
self.wheel_or_sdist_by_package = {}
79+
self.resolved_requirements = resolved_requirements
7980

8081
def identify(self, requirement_or_candidate):
8182
"""Given a requirement, return an identifier for it. Overridden."""
@@ -118,29 +119,31 @@ def get_versions_for_package_from_repo(self, name, repo):
118119
python_version = packaging.version.parse(
119120
get_python_version_from_env_tag(self.environment.python_version)
120121
)
122+
formats = []
121123
wheels = list(package.get_supported_wheels(environment=self.environment))
122124
if wheels:
123125
valid_wheel_present = False
124126
for wheel in wheels:
125-
if wheel.requires_python and python_version in SpecifierSet(
127+
if (
126128
wheel.requires_python
127-
):
128-
valid_wheel_present = True
129-
if not wheel.requires_python:
129+
and python_version in SpecifierSet(wheel.requires_python)
130+
) or not wheel.requires_python:
130131
valid_wheel_present = True
131132
if valid_wheel_present:
132-
self.wheel_or_sdist_by_package[str(purl)] = "Wheel"
133133
versions.append(version)
134-
continue
134+
formats.append("Wheel")
135135
if package.sdist:
136-
if package.sdist.requires_python and python_version in SpecifierSet(
136+
valid_sdist_present = False
137+
if (
137138
package.sdist.requires_python
139+
and python_version in SpecifierSet(package.sdist.requires_python)
140+
or not package.sdist.requires_python
138141
):
139-
self.wheel_or_sdist_by_package[str(purl)] = "Sdist"
140-
versions.append(version)
141-
if not package.sdist.requires_python:
142-
self.wheel_or_sdist_by_package[str(purl)] = "Sdist"
142+
valid_sdist_present = True
143+
if valid_sdist_present:
143144
versions.append(version)
145+
formats.append("Sdist")
146+
self.wheel_or_sdist_by_package[str(purl)] = formats
144147
return versions
145148

146149
def get_versions_for_package_from_pypi_json_api(self, name):
@@ -170,95 +173,110 @@ def get_requirements_for_package_from_pypi_simple(self, candidate):
170173

171174
purl = PackageURL(type="pypi", name=candidate.name, version=str(candidate.version))
172175

173-
wheel_or_sdist = self.wheel_or_sdist_by_package[str(purl)]
176+
formats = self.wheel_or_sdist_by_package[str(purl)]
174177

175-
if wheel_or_sdist == "Wheel":
176-
wheels = utils_pypi.download_wheel(
177-
name=candidate.name,
178-
version=str(candidate.version),
179-
environment=self.environment,
180-
repos=self.repos,
181-
)
182-
for wheel in wheels:
183-
deps = list(
184-
PypiWheelHandler.parse(os.path.join(utils_pypi.CACHE_THIRDPARTY_DIR, wheel))
178+
for format in formats:
179+
if format == "Wheel":
180+
wheels = utils_pypi.download_wheel(
181+
name=candidate.name,
182+
version=str(candidate.version),
183+
environment=self.environment,
184+
repos=self.repos,
185185
)
186-
assert len(deps) == 1
187-
deps = deps[0].dependencies
188-
for dep in deps:
189-
if dep.scope == "install":
190-
yield packaging.requirements.Requirement(str(dep.extracted_requirement))
191-
192-
if wheel_or_sdist == "Sdist":
193-
sdist = utils_pypi.download_sdist(
194-
name=candidate.name,
195-
version=str(candidate.version),
196-
repos=self.repos,
197-
)
186+
for wheel in wheels:
187+
deps = list(
188+
PypiWheelHandler.parse(os.path.join(utils_pypi.CACHE_THIRDPARTY_DIR, wheel))
189+
)
190+
assert len(deps) == 1
191+
deps = deps[0].dependencies
192+
for dep in deps:
193+
if dep.scope == "install":
194+
yield packaging.requirements.Requirement(str(dep.extracted_requirement))
198195

199-
if sdist.endswith(".tar.gz"):
200-
sdist_file = sdist.rstrip(".tar.gz")
201-
file = tarfile.open(os.path.join(utils_pypi.CACHE_THIRDPARTY_DIR, sdist))
202-
file.extractall(
203-
os.path.join(utils_pypi.CACHE_THIRDPARTY_DIR, "extracted_sdists", sdist_file)
196+
if format == "Sdist":
197+
sdist = utils_pypi.download_sdist(
198+
name=candidate.name,
199+
version=str(candidate.version),
200+
repos=self.repos,
204201
)
205-
file.close()
206-
if sdist.endswith(".zip"):
207-
sdist_file = sdist.rstrip(".zip")
208-
with ZipFile(os.path.join(utils_pypi.CACHE_THIRDPARTY_DIR, sdist), "r") as zip:
209-
zip.extractall(
202+
if sdist.endswith(".tar.gz"):
203+
sdist_file = sdist.rstrip(".tar.gz")
204+
file = tarfile.open(os.path.join(utils_pypi.CACHE_THIRDPARTY_DIR, sdist))
205+
file.extractall(
210206
os.path.join(
211207
utils_pypi.CACHE_THIRDPARTY_DIR, "extracted_sdists", sdist_file
212208
)
213209
)
214-
setup_py_path = os.path.join(
215-
utils_pypi.CACHE_THIRDPARTY_DIR,
216-
"extracted_sdists",
217-
sdist_file,
218-
sdist_file,
219-
"setup.py",
220-
)
221-
setup_cfg_path = os.path.join(
222-
utils_pypi.CACHE_THIRDPARTY_DIR,
223-
"extracted_sdists",
224-
sdist_file,
225-
sdist_file,
226-
"setup.cfg",
227-
)
228-
pkg_info_path = os.path.join(
229-
utils_pypi.CACHE_THIRDPARTY_DIR, "extracted_sdists", sdist_file
230-
)
231-
requirement_path = os.path.join(
232-
utils_pypi.CACHE_THIRDPARTY_DIR,
233-
"extracted_sdists",
234-
sdist_file,
235-
sdist_file,
236-
"requirements.txt",
237-
)
210+
file.close()
211+
if sdist.endswith(".zip"):
212+
sdist_file = sdist.rstrip(".zip")
213+
with ZipFile(os.path.join(utils_pypi.CACHE_THIRDPARTY_DIR, sdist), "r") as zip:
214+
zip.extractall(
215+
os.path.join(
216+
utils_pypi.CACHE_THIRDPARTY_DIR, "extracted_sdists", sdist_file
217+
)
218+
)
219+
setup_py_path = os.path.join(
220+
utils_pypi.CACHE_THIRDPARTY_DIR,
221+
"extracted_sdists",
222+
sdist_file,
223+
sdist_file,
224+
"setup.py",
225+
)
226+
setup_cfg_path = os.path.join(
227+
utils_pypi.CACHE_THIRDPARTY_DIR,
228+
"extracted_sdists",
229+
sdist_file,
230+
sdist_file,
231+
"setup.cfg",
232+
)
233+
pkg_info_path = os.path.join(
234+
utils_pypi.CACHE_THIRDPARTY_DIR, "extracted_sdists", sdist_file
235+
)
236+
requirement_path = os.path.join(
237+
utils_pypi.CACHE_THIRDPARTY_DIR,
238+
"extracted_sdists",
239+
sdist_file,
240+
sdist_file,
241+
"requirements.txt",
242+
)
238243

239-
path_by_format = {
240-
"pkginfo": pkg_info_path,
241-
"setup-py": setup_py_path,
242-
"setup-cfg": setup_cfg_path,
243-
"requirement": requirement_path,
244-
}
245-
246-
handler_by_format = {
247-
"pkginfo": PythonSdistPkgInfoFile,
248-
"setup-py": PythonSetupPyHandler,
249-
"setup-cfg": SetupCfgHandler,
250-
"requirement": PipRequirementsFileHandler,
251-
}
252-
for format in ["pkginfo", "setup-py", "setup-cfg", "requirement"]:
253-
path = path_by_format[format]
254-
if os.path.exists(path):
255-
handler = handler_by_format[format]
256-
deps = list(handler.parse(path))
257-
assert len(deps) == 1
258-
dependencies = deps[0].dependencies
259-
for dep in dependencies:
260-
if dep.scope == "install":
261-
yield packaging.requirements.Requirement(str(dep.extracted_requirement))
244+
path_by_format = {
245+
"pkginfo": pkg_info_path,
246+
"setup-py": setup_py_path,
247+
"setup-cfg": setup_cfg_path,
248+
"requirement": requirement_path,
249+
}
250+
251+
handler_by_format = {
252+
"pkginfo": PythonSdistPkgInfoFile,
253+
"setup-py": PythonSetupPyHandler,
254+
"setup-cfg": SetupCfgHandler,
255+
"requirement": PipRequirementsFileHandler,
256+
}
257+
for format in ["pkginfo", "setup-py", "setup-cfg", "requirement"]:
258+
path = path_by_format[format]
259+
if os.path.exists(path):
260+
handler = handler_by_format[format]
261+
deps = list(handler.parse(path))
262+
assert len(deps) == 1
263+
dependencies = deps[0].dependencies
264+
for dep in dependencies:
265+
if dep.purl:
266+
dep_purl = PackageURL.from_string(dep.purl)
267+
if dep.scope == "install" and (
268+
not (dep.is_resolved)
269+
or (
270+
dep.is_resolved
271+
and dep_purl.name not in self.resolved_requirements
272+
)
273+
):
274+
if dep.is_resolved:
275+
self.resolved_requirements.append(dep_purl)
276+
if not dep.extracted_requirement.startswith("--"):
277+
yield packaging.requirements.Requirement(
278+
str(dep.extracted_requirement)
279+
)
262280

263281
def get_requirements_for_package_from_pypi_json_api(self, purl):
264282

@@ -433,8 +451,16 @@ def get_resolved_dependencies(
433451
Used the provided ``repos`` list of PypiSimpleRepository.
434452
If empty, use instead the PyPI.org JSON API exclusively instead
435453
"""
454+
resolved_requirements = [
455+
packaging.utils.canonicalize_name(r["requirement"].name)
456+
for r in requirements
457+
if r["is_requirement_resolved"]
458+
]
459+
requirements = [r["requirement"] for r in requirements]
436460
resolver = Resolver(
437-
provider=PythonInputProvider(environment=environment, repos=repos),
461+
provider=PythonInputProvider(
462+
environment=environment, repos=repos, resolved_requirements=resolved_requirements
463+
),
438464
reporter=BaseReporter(),
439465
)
440466
results = resolver.resolve(requirements=requirements, max_rounds=max_rounds)

‎src/python_inspector/resolve_cli.py‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -274,7 +274,11 @@ def resolve(direct_dependencies, environment, repos=tuple(), as_tree=False, max_
274274
If empty, use instead the PyPI.org JSON API exclusively.
275275
"""
276276
requirements = [
277-
Requirement(requirement_string=d.extracted_requirement) for d in direct_dependencies
277+
dict(
278+
requirement=Requirement(requirement_string=d.extracted_requirement),
279+
is_requirement_resolved=d.is_resolved,
280+
)
281+
for d in direct_dependencies
278282
]
279283
resolved_dependencies = get_resolved_dependencies(
280284
requirements=requirements,

‎tests/data/pinned-requirements.txt-expected.json‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -273,6 +273,10 @@
273273
"package": "pkg:pypi/certifi@2022.5.18.1",
274274
"dependencies": []
275275
},
276+
{
277+
"package": "pkg:pypi/chardet@4.0.0",
278+
"dependencies": []
279+
},
276280
{
277281
"package": "pkg:pypi/charset-normalizer@2.0.12",
278282
"dependencies": []
@@ -290,10 +294,12 @@
290294
"dependencies": [
291295
"pkg:pypi/attrs@21.4.0",
292296
"pkg:pypi/beautifulsoup4@4.11.1",
297+
"pkg:pypi/chardet@4.0.0",
293298
"pkg:pypi/click@8.0.4",
294299
"pkg:pypi/intbitset@3.0.1",
295300
"pkg:pypi/requests@2.27.1",
296301
"pkg:pypi/saneyaml@0.5.2",
302+
"pkg:pypi/six@1.16.0",
297303
"pkg:pypi/text-unidecode@1.3"
298304
]
299305
},
@@ -366,6 +372,10 @@
366372
"pkg:pypi/pyyaml@6.0"
367373
]
368374
},
375+
{
376+
"package": "pkg:pypi/six@1.16.0",
377+
"dependencies": []
378+
},
369379
{
370380
"package": "pkg:pypi/soupsieve@2.3.2.post1",
371381
"dependencies": []

‎tests/test_resolution.py‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020

2121
@pytest.mark.online
2222
def test_get_resolved_dependencies_with_flask_and_python_310():
23-
req = [Requirement("flask==2.1.2")]
23+
req = [dict(requirement=Requirement("flask==2.1.2"), is_requirement_resolved=True)]
2424
results = get_resolved_dependencies(
2525
requirements=req,
2626
environment=Environment(
@@ -43,7 +43,7 @@ def test_get_resolved_dependencies_with_flask_and_python_310():
4343

4444
@pytest.mark.online
4545
def test_get_resolved_dependencies_with_flask_and_python_310_windows():
46-
req = [Requirement("flask==2.1.2")]
46+
req = [dict(requirement=Requirement("flask==2.1.2"), is_requirement_resolved=True)]
4747
results = get_resolved_dependencies(
4848
requirements=req,
4949
environment=Environment(
@@ -67,7 +67,7 @@ def test_get_resolved_dependencies_with_flask_and_python_310_windows():
6767

6868
@pytest.mark.online
6969
def test_get_resolved_dependencies_with_flask_and_python_36():
70-
req = [Requirement("flask")]
70+
req = [dict(requirement=Requirement("flask"), is_requirement_resolved=False)]
7171
results = get_resolved_dependencies(
7272
requirements=req,
7373
environment=Environment(
@@ -95,7 +95,7 @@ def test_get_resolved_dependencies_with_flask_and_python_36():
9595

9696
@pytest.mark.online
9797
def test_get_resolved_dependencies_with_tilde_requirement_using_json_api():
98-
req = [Requirement("flask~=2.1.2")]
98+
req = [dict(requirement=Requirement("flask~=2.1.2"), is_requirement_resolved=True)]
9999
results = get_resolved_dependencies(
100100
requirements=req,
101101
as_tree=False,
@@ -119,7 +119,7 @@ def test_get_resolved_dependencies_with_tilde_requirement_using_json_api():
119119

120120
@pytest.mark.online
121121
def test_without_supported_wheels():
122-
req = [Requirement("autobahn==22.3.2")]
122+
req = [dict(requirement=Requirement("autobahn==22.3.2"), is_requirement_resolved=True)]
123123
results = get_resolved_dependencies(
124124
requirements=req,
125125
as_tree=False,
@@ -134,11 +134,11 @@ def test_without_supported_wheels():
134134
assert as_list == [
135135
"pkg:pypi/autobahn@22.3.2",
136136
"pkg:pypi/cffi@1.15.1",
137-
"pkg:pypi/cryptography@37.0.3",
137+
"pkg:pypi/cryptography@37.0.4",
138138
"pkg:pypi/hyperlink@21.0.0",
139139
"pkg:pypi/idna@3.3",
140140
"pkg:pypi/pycparser@2.21",
141-
"pkg:pypi/setuptools@62.6.0",
141+
"pkg:pypi/setuptools@63.1.0",
142142
"pkg:pypi/txaio@22.2.1",
143143
]
144144

0 commit comments

Comments
 (0)