Skip to content

Commit fd73eed

Browse files
TG1999TG1999
committed
Ignore resolved dependencies
Signed-off-by: Tushar Goel <tushar.goel.dav@gmail.com>
1 parent 1047a8a commit fd73eed

5 files changed

Lines changed: 156 additions & 106 deletions

File tree

‎src/_packagedcode/pypi.py‎

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -745,7 +745,8 @@ def parse_reqs(cls, reqs, scope):
745745
for req in reqs:
746746
is_resolved = False
747747
req_parsed = packaging.requirements.Requirement(str(req))
748-
purl = PackageURL(type="pypi", name=req_parsed.name)
748+
name = canonicalize_name(req_parsed.name)
749+
purl = PackageURL(type="pypi", name=name)
749750
specifiers = req_parsed.specifier._specs
750751
if len(specifiers) == 1:
751752
specifier = list(specifiers)[0]
@@ -754,7 +755,7 @@ def parse_reqs(cls, reqs, scope):
754755
purl = purl._replace(version=specifier.version)
755756
dependent_packages.append(
756757
models.DependentPackage(
757-
purl=purl,
758+
purl=str(purl),
758759
scope=scope,
759760
is_runtime=True,
760761
is_optional=False,
@@ -1932,18 +1933,24 @@ def compute_normalized_license(declared_license):
19321933

19331934

19341935
def get_requirement_from_section(section, sub_section):
1936+
"""
1937+
Generate requirements from the `sub_section`
1938+
"""
19351939
content = section.get(sub_section)
19361940
if content:
19371941
for req in content.splitlines():
19381942
if req:
1943+
#pytest-mypy >= 0.9.1; \
19391944
req = req.replace("; \\", "")
1945+
# pip>=19.1 # For proper file:// URLs support.
19401946
if "#" in req:
19411947
req , _ = req.rsplit("#")
1948+
#pure-eval; black; tox;
19421949
req_split_by_semi_colon = req.split(";")
19431950
req_split_by_semi_colon = [req.strip() for req in req_split_by_semi_colon]
1944-
if len(req_split_by_semi_colon) >= 2 and not(req_split_by_semi_colon[1].startswith("python_version")
1945-
or req_split_by_semi_colon[1].startswith("sys_platform")
1946-
or req_split_by_semi_colon[1].startswith("platform_system")):
1951+
if len(req_split_by_semi_colon) >= 2 and not(req_split_by_semi_colon[1].startswith("python_version") # pip>=19.1 ;python_version > 3.7
1952+
or req_split_by_semi_colon[1].startswith("sys_platform") # pip>=19.1 ;sys_platform = "Windows"
1953+
or req_split_by_semi_colon[1].startswith("platform_system")): # pip>=19.1 ;platform_system = "Windows"
19471954
for temp_req in req_split_by_semi_colon:
19481955
yield temp_req
19491956
else:

‎src/python_inspector/resolution.py‎

Lines changed: 122 additions & 93 deletions
Original file line numberDiff line numberDiff line change
@@ -70,12 +70,13 @@ def get_python_version_from_env_tag(python_version: str):
7070

7171

7272
class PythonInputProvider(AbstractProvider):
73-
def __init__(self, environment=None, repos=tuple()):
73+
def __init__(self, environment=None, repos=tuple(), resolved_requirements=[]):
7474
self.environment = environment
7575
self.repos = repos or []
7676
self.versions_by_package = {}
7777
self.dependencies_by_purl = {}
7878
self.wheel_or_sdist_by_package = {}
79+
self.resolved_requirements = resolved_requirements
7980

8081
def identify(self, requirement_or_candidate):
8182
"""Given a requirement, return an identifier for it. Overridden."""
@@ -118,29 +119,31 @@ def get_versions_for_package_from_repo(self, name, repo):
118119
python_version = packaging.version.parse(
119120
get_python_version_from_env_tag(self.environment.python_version)
120121
)
122+
formats = []
121123
wheels = list(package.get_supported_wheels(environment=self.environment))
122124
if wheels:
123125
valid_wheel_present = False
124126
for wheel in wheels:
125-
if wheel.requires_python and python_version in SpecifierSet(
127+
if (
126128
wheel.requires_python
127-
):
128-
valid_wheel_present = True
129-
if not wheel.requires_python:
129+
and python_version in SpecifierSet(wheel.requires_python)
130+
) or not wheel.requires_python:
130131
valid_wheel_present = True
131132
if valid_wheel_present:
132-
self.wheel_or_sdist_by_package[str(purl)] = "Wheel"
133133
versions.append(version)
134-
continue
134+
formats.append("Wheel")
135135
if package.sdist:
136-
if package.sdist.requires_python and python_version in SpecifierSet(
136+
valid_sdist_present = False
137+
if (
137138
package.sdist.requires_python
139+
and python_version in SpecifierSet(package.sdist.requires_python)
140+
or not package.sdist.requires_python
138141
):
139-
self.wheel_or_sdist_by_package[str(purl)] = "Sdist"
140-
versions.append(version)
141-
if not package.sdist.requires_python:
142-
self.wheel_or_sdist_by_package[str(purl)] = "Sdist"
142+
valid_sdist_present = True
143+
if valid_sdist_present:
143144
versions.append(version)
145+
formats.append("Sdist")
146+
self.wheel_or_sdist_by_package[str(purl)] = formats
144147
return versions
145148

146149
def get_versions_for_package_from_pypi_json_api(self, name):
@@ -170,95 +173,113 @@ def get_requirements_for_package_from_pypi_simple(self, candidate):
170173

171174
purl = PackageURL(type="pypi", name=candidate.name, version=str(candidate.version))
172175

173-
wheel_or_sdist = self.wheel_or_sdist_by_package[str(purl)]
176+
formats = self.wheel_or_sdist_by_package[str(purl)]
174177

175-
if wheel_or_sdist == "Wheel":
176-
wheels = utils_pypi.download_wheel(
177-
name=candidate.name,
178-
version=str(candidate.version),
179-
environment=self.environment,
180-
repos=self.repos,
181-
)
182-
for wheel in wheels:
183-
deps = list(
184-
PypiWheelHandler.parse(os.path.join(utils_pypi.CACHE_THIRDPARTY_DIR, wheel))
178+
for format in formats:
179+
if format == "Wheel":
180+
wheels = utils_pypi.download_wheel(
181+
name=candidate.name,
182+
version=str(candidate.version),
183+
environment=self.environment,
184+
repos=self.repos,
185185
)
186-
assert len(deps) == 1
187-
deps = deps[0].dependencies
188-
for dep in deps:
189-
if dep.scope == "install":
190-
yield packaging.requirements.Requirement(str(dep.extracted_requirement))
191-
192-
if wheel_or_sdist == "Sdist":
193-
sdist = utils_pypi.download_sdist(
194-
name=candidate.name,
195-
version=str(candidate.version),
196-
repos=self.repos,
197-
)
186+
for wheel in wheels:
187+
deps = list(
188+
PypiWheelHandler.parse(os.path.join(utils_pypi.CACHE_THIRDPARTY_DIR, wheel))
189+
)
190+
assert len(deps) == 1
191+
deps = deps[0].dependencies
192+
for dep in deps:
193+
if dep.scope == "install":
194+
yield packaging.requirements.Requirement(str(dep.extracted_requirement))
198195

199-
if sdist.endswith(".tar.gz"):
200-
sdist_file = sdist.rstrip(".tar.gz")
201-
file = tarfile.open(os.path.join(utils_pypi.CACHE_THIRDPARTY_DIR, sdist))
202-
file.extractall(
203-
os.path.join(utils_pypi.CACHE_THIRDPARTY_DIR, "extracted_sdists", sdist_file)
196+
if format == "Sdist":
197+
sdist = utils_pypi.download_sdist(
198+
name=candidate.name,
199+
version=str(candidate.version),
200+
repos=self.repos,
204201
)
205-
file.close()
206-
if sdist.endswith(".zip"):
207-
sdist_file = sdist.rstrip(".zip")
208-
with ZipFile(os.path.join(utils_pypi.CACHE_THIRDPARTY_DIR, sdist), "r") as zip:
209-
zip.extractall(
202+
if sdist.endswith(".tar.gz"):
203+
sdist_file = sdist.rstrip(".tar.gz")
204+
file = tarfile.open(os.path.join(utils_pypi.CACHE_THIRDPARTY_DIR, sdist))
205+
file.extractall(
210206
os.path.join(
211207
utils_pypi.CACHE_THIRDPARTY_DIR, "extracted_sdists", sdist_file
212208
)
213209
)
214-
setup_py_path = os.path.join(
215-
utils_pypi.CACHE_THIRDPARTY_DIR,
216-
"extracted_sdists",
217-
sdist_file,
218-
sdist_file,
219-
"setup.py",
220-
)
221-
setup_cfg_path = os.path.join(
222-
utils_pypi.CACHE_THIRDPARTY_DIR,
223-
"extracted_sdists",
224-
sdist_file,
225-
sdist_file,
226-
"setup.cfg",
227-
)
228-
pkg_info_path = os.path.join(
229-
utils_pypi.CACHE_THIRDPARTY_DIR, "extracted_sdists", sdist_file
230-
)
231-
requirement_path = os.path.join(
232-
utils_pypi.CACHE_THIRDPARTY_DIR,
233-
"extracted_sdists",
234-
sdist_file,
235-
sdist_file,
236-
"requirements.txt",
237-
)
210+
file.close()
211+
if sdist.endswith(".zip"):
212+
sdist_file = sdist.rstrip(".zip")
213+
with ZipFile(os.path.join(utils_pypi.CACHE_THIRDPARTY_DIR, sdist), "r") as zip:
214+
zip.extractall(
215+
os.path.join(
216+
utils_pypi.CACHE_THIRDPARTY_DIR, "extracted_sdists", sdist_file
217+
)
218+
)
219+
setup_py_path = os.path.join(
220+
utils_pypi.CACHE_THIRDPARTY_DIR,
221+
"extracted_sdists",
222+
sdist_file,
223+
sdist_file,
224+
"setup.py",
225+
)
226+
setup_cfg_path = os.path.join(
227+
utils_pypi.CACHE_THIRDPARTY_DIR,
228+
"extracted_sdists",
229+
sdist_file,
230+
sdist_file,
231+
"setup.cfg",
232+
)
233+
pkg_info_path = os.path.join(
234+
utils_pypi.CACHE_THIRDPARTY_DIR, "extracted_sdists", sdist_file
235+
)
236+
requirement_path = os.path.join(
237+
utils_pypi.CACHE_THIRDPARTY_DIR,
238+
"extracted_sdists",
239+
sdist_file,
240+
sdist_file,
241+
"requirements.txt",
242+
)
238243

239-
path_by_format = {
240-
"pkginfo": pkg_info_path,
241-
"setup-py": setup_py_path,
242-
"setup-cfg": setup_cfg_path,
243-
"requirement": requirement_path,
244-
}
245-
246-
handler_by_format = {
247-
"pkginfo": PythonSdistPkgInfoFile,
248-
"setup-py": PythonSetupPyHandler,
249-
"setup-cfg": SetupCfgHandler,
250-
"requirement": PipRequirementsFileHandler,
251-
}
252-
for format in ["pkginfo", "setup-py", "setup-cfg", "requirement"]:
253-
path = path_by_format[format]
254-
if os.path.exists(path):
255-
handler = handler_by_format[format]
256-
deps = list(handler.parse(path))
257-
assert len(deps) == 1
258-
dependencies = deps[0].dependencies
259-
for dep in dependencies:
260-
if dep.scope == "install":
261-
yield packaging.requirements.Requirement(str(dep.extracted_requirement))
244+
path_by_format = {
245+
"pkginfo": pkg_info_path,
246+
"setup-py": setup_py_path,
247+
"setup-cfg": setup_cfg_path,
248+
"requirement": requirement_path,
249+
}
250+
251+
handler_by_format = {
252+
"pkginfo": PythonSdistPkgInfoFile,
253+
"setup-py": PythonSetupPyHandler,
254+
"setup-cfg": SetupCfgHandler,
255+
"requirement": PipRequirementsFileHandler,
256+
}
257+
for format in ["pkginfo", "setup-py", "setup-cfg", "requirement"]:
258+
path = path_by_format[format]
259+
if os.path.exists(path):
260+
handler = handler_by_format[format]
261+
deps = list(handler.parse(path))
262+
assert len(deps) == 1
263+
dependencies = deps[0].dependencies
264+
for dep in dependencies:
265+
# skip if no purl can be extracted for dependency
266+
if dep.purl:
267+
dep_purl = PackageURL.from_string(dep.purl)
268+
if dep.scope == "install" and (
269+
not (dep.is_resolved)
270+
or (
271+
dep.is_resolved
272+
and dep_purl.name not in self.resolved_requirements
273+
)
274+
):
275+
if dep.is_resolved:
276+
self.resolved_requirements.append(dep_purl)
277+
# skip the requirement starting with -- like
278+
# --editable, --requirement
279+
if not dep.extracted_requirement.startswith("--"):
280+
yield packaging.requirements.Requirement(
281+
str(dep.extracted_requirement)
282+
)
262283

263284
def get_requirements_for_package_from_pypi_json_api(self, purl):
264285

@@ -433,8 +454,16 @@ def get_resolved_dependencies(
433454
Used the provided ``repos`` list of PypiSimpleRepository.
434455
If empty, use instead the PyPI.org JSON API exclusively instead
435456
"""
457+
resolved_requirements = [
458+
packaging.utils.canonicalize_name(r["requirement"].name)
459+
for r in requirements
460+
if r["is_requirement_resolved"]
461+
]
462+
requirements = [r["requirement"] for r in requirements]
436463
resolver = Resolver(
437-
provider=PythonInputProvider(environment=environment, repos=repos),
464+
provider=PythonInputProvider(
465+
environment=environment, repos=repos, resolved_requirements=resolved_requirements
466+
),
438467
reporter=BaseReporter(),
439468
)
440469
results = resolver.resolve(requirements=requirements, max_rounds=max_rounds)

‎src/python_inspector/resolve_cli.py‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -274,7 +274,11 @@ def resolve(direct_dependencies, environment, repos=tuple(), as_tree=False, max_
274274
If empty, use instead the PyPI.org JSON API exclusively.
275275
"""
276276
requirements = [
277-
Requirement(requirement_string=d.extracted_requirement) for d in direct_dependencies
277+
dict(
278+
requirement=Requirement(requirement_string=d.extracted_requirement),
279+
is_requirement_resolved=d.is_resolved,
280+
)
281+
for d in direct_dependencies
278282
]
279283
resolved_dependencies = get_resolved_dependencies(
280284
requirements=requirements,

‎tests/data/pinned-requirements.txt-expected.json‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -273,6 +273,10 @@
273273
"package": "pkg:pypi/certifi@2022.5.18.1",
274274
"dependencies": []
275275
},
276+
{
277+
"package": "pkg:pypi/chardet@4.0.0",
278+
"dependencies": []
279+
},
276280
{
277281
"package": "pkg:pypi/charset-normalizer@2.0.12",
278282
"dependencies": []
@@ -290,10 +294,12 @@
290294
"dependencies": [
291295
"pkg:pypi/attrs@21.4.0",
292296
"pkg:pypi/beautifulsoup4@4.11.1",
297+
"pkg:pypi/chardet@4.0.0",
293298
"pkg:pypi/click@8.0.4",
294299
"pkg:pypi/intbitset@3.0.1",
295300
"pkg:pypi/requests@2.27.1",
296301
"pkg:pypi/saneyaml@0.5.2",
302+
"pkg:pypi/six@1.16.0",
297303
"pkg:pypi/text-unidecode@1.3"
298304
]
299305
},
@@ -366,6 +372,10 @@
366372
"pkg:pypi/pyyaml@6.0"
367373
]
368374
},
375+
{
376+
"package": "pkg:pypi/six@1.16.0",
377+
"dependencies": []
378+
},
369379
{
370380
"package": "pkg:pypi/soupsieve@2.3.2.post1",
371381
"dependencies": []

0 commit comments

Comments
 (0)