Skip to content

Commit 7749189

Browse files
TG1999TG1999
committed
Add extra data and fix resolution
Signed-off-by: Tushar Goel <tushar.goel.dav@gmail.com>
1 parent 9ae2874 commit 7749189

12 files changed

Lines changed: 1098 additions & 68 deletions

‎src/_packagedcode/models.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -373,6 +373,11 @@ class DependentPackage(ModelMixin):
373373
'either from the datafile or collected from another source. Some '
374374
'lockfiles for Composer or Cargo contain extra dependency data.'
375375
)
376+
377+
extra_data = Mapping(
378+
label='extra data',
379+
help='A mapping of arbitrary extra data.',
380+
)
376381

377382

378383
@attr.attributes(slots=True)

‎src/_packagedcode/pypi.py‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -921,7 +921,7 @@ def get_requirements_txt_dependencies(location, include_nested=False):
921921
if req.name:
922922
# will be None if not pinned
923923
version = req.get_pinned_version
924-
purl = PackageURL(type='pypi', name=req.name, version=version)
924+
purl = PackageURL(type='pypi', name=canonicalize_name(req.name), version=version)
925925

926926
else:
927927
# this is odd, but this can be null
@@ -954,6 +954,18 @@ def get_requirements_txt_dependencies(location, include_nested=False):
954954
is_optional=is_optional,
955955
is_resolved=req.is_pinned or False,
956956
extracted_requirement=requirement,
957+
extra_data=dict(
958+
is_editable=req.is_editable,
959+
link=req.link and req.link.url or None,
960+
hash_options=req.hash_options or [],
961+
is_constraint=req.is_constraint,
962+
is_archive=req.is_archive,
963+
is_wheel=req.is_wheel,
964+
is_url=req.is_url,
965+
is_vcs_url=req.is_vcs_url,
966+
is_name_at_url=req.is_name_at_url,
967+
is_local_path=req.is_local_path,
968+
),
957969
)
958970
)
959971

‎src/python_inspector/resolution.py‎

Lines changed: 13 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -277,18 +277,23 @@ def get_requirements_for_package_from_pypi_simple(self, candidate):
277277

278278
dep_purl = PackageURL.from_string(dep.purl)
279279

280-
if self.is_dep_resolved_and_in_resolved_requirements(dep, dep_purl):
280+
dep_purl_name = packaging.utils.canonicalize_name(dep_purl.name)
281+
282+
if self.is_dep_resolved_and_in_resolved_requirements(dep, dep_purl_name):
283+
yield packaging.requirements.Requirement(
284+
f"{str(dep_purl_name)}{str(self.resolved_requirements[str(dep_purl_name)])}"
285+
)
281286
continue
282287

283288
if dep.is_resolved:
284-
self.resolved_requirements.append(dep_purl)
289+
self.resolved_requirements[dep_purl_name] = f"=={dep_purl.version}"
285290
# skip the requirement starting with -- like
286291
# --editable, --requirement
287-
if not dep.extracted_requirement.startswith("--"):
292+
if not dep.extracted_requirement.startswith("-"):
288293
yield packaging.requirements.Requirement(str(dep.extracted_requirement))
289294

290-
def is_dep_resolved_and_in_resolved_requirements(self, dep, dep_purl):
291-
return dep.is_resolved and dep_purl.name in self.resolved_requirements
295+
def is_dep_resolved_and_in_resolved_requirements(self, dep, dep_purl_name):
296+
return dep.is_resolved and dep_purl_name in self.resolved_requirements
292297

293298
def get_requirements_for_package_from_pypi_json_api(self, purl):
294299
"""
@@ -516,11 +521,11 @@ def get_resolved_dependencies(
516521
Used the provided ``repos`` list of PypiSimpleRepository.
517522
If empty, use instead the PyPI.org JSON API exclusively instead
518523
"""
519-
resolved_requirements = [
520-
packaging.utils.canonicalize_name(r.name)
524+
resolved_requirements = {
525+
packaging.utils.canonicalize_name(r.name): r.specifier
521526
for r in requirements
522527
if getattr(r, "is_requirement_resolved", False)
523-
]
528+
}
524529
resolver = Resolver(
525530
provider=PythonInputProvider(
526531
environment=environment, repos=repos, resolved_requirements=resolved_requirements

‎src/python_inspector/resolve_cli.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -280,6 +280,9 @@ def resolve(direct_dependencies, environment, repos=tuple(), as_tree=False, max_
280280
requirements = []
281281

282282
for dependency in direct_dependencies:
283+
# FIXME We are skipping editable requirements for now
284+
if dependency.extra_data.get("is_editable"):
285+
continue
283286
requirement = Requirement(requirement_string=dependency.extracted_requirement)
284287
requirement.is_requirement_resolved = dependency.is_resolved
285288
requirements.append(requirement)

‎tests/data/default-url-expected.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,8 @@
2222
"is_runtime": true,
2323
"is_optional": false,
2424
"is_resolved": true,
25-
"resolved_package": {}
25+
"resolved_package": {},
26+
"extra_data": {}
2627
}
2728
],
2829
"resolved_dependencies": [

‎tests/data/parse-reqs-with-setup_requires-and-python-requires.json‎

Lines changed: 18 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,8 @@
3636
"is_runtime": true,
3737
"is_optional": false,
3838
"is_resolved": false,
39-
"resolved_package": {}
39+
"resolved_package": {},
40+
"extra_data": {}
4041
},
4142
{
4243
"purl": "pkg:pypi/qtpy",
@@ -45,7 +46,8 @@
4546
"is_runtime": true,
4647
"is_optional": false,
4748
"is_resolved": false,
48-
"resolved_package": {}
49+
"resolved_package": {},
50+
"extra_data": {}
4951
},
5052
{
5153
"purl": "pkg:pypi/setuptools",
@@ -54,7 +56,8 @@
5456
"is_runtime": true,
5557
"is_optional": false,
5658
"is_resolved": false,
57-
"resolved_package": {}
59+
"resolved_package": {},
60+
"extra_data": {}
5861
},
5962
{
6063
"purl": "pkg:generic/python",
@@ -63,7 +66,8 @@
6366
"is_runtime": true,
6467
"is_optional": false,
6568
"is_resolved": false,
66-
"resolved_package": {}
69+
"resolved_package": {},
70+
"extra_data": {}
6771
},
6872
{
6973
"purl": "pkg:pypi/pyside2",
@@ -72,7 +76,8 @@
7276
"is_runtime": true,
7377
"is_optional": false,
7478
"is_resolved": false,
75-
"resolved_package": {}
79+
"resolved_package": {},
80+
"extra_data": {}
7681
},
7782
{
7883
"purl": "pkg:pypi/pyqt5",
@@ -81,7 +86,8 @@
8186
"is_runtime": true,
8287
"is_optional": false,
8388
"is_resolved": false,
84-
"resolved_package": {}
89+
"resolved_package": {},
90+
"extra_data": {}
8591
},
8692
{
8793
"purl": "pkg:pypi/sphinx",
@@ -90,7 +96,8 @@
9096
"is_runtime": true,
9197
"is_optional": false,
9298
"is_resolved": false,
93-
"resolved_package": {}
99+
"resolved_package": {},
100+
"extra_data": {}
94101
},
95102
{
96103
"purl": "pkg:pypi/restructuredtext-lint",
@@ -99,7 +106,8 @@
99106
"is_runtime": true,
100107
"is_optional": false,
101108
"is_resolved": false,
102-
"resolved_package": {}
109+
"resolved_package": {},
110+
"extra_data": {}
103111
},
104112
{
105113
"purl": "pkg:pypi/pyside2",
@@ -108,7 +116,8 @@
108116
"is_runtime": true,
109117
"is_optional": false,
110118
"is_resolved": false,
111-
"resolved_package": {}
119+
"resolved_package": {},
120+
"extra_data": {}
112121
}
113122
],
114123
"repository_homepage_url": null,

‎tests/data/parse-reqs.json‎

Lines changed: 34 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,8 @@
3636
"is_runtime": true,
3737
"is_optional": false,
3838
"is_resolved": false,
39-
"resolved_package": {}
39+
"resolved_package": {},
40+
"extra_data": {}
4041
},
4142
{
4243
"purl": "pkg:pypi/attrs",
@@ -45,7 +46,8 @@
4546
"is_runtime": true,
4647
"is_optional": false,
4748
"is_resolved": false,
48-
"resolved_package": {}
49+
"resolved_package": {},
50+
"extra_data": {}
4951
},
5052
{
5153
"purl": "pkg:pypi/beautifulsoup4",
@@ -54,7 +56,8 @@
5456
"is_runtime": true,
5557
"is_optional": false,
5658
"is_resolved": false,
57-
"resolved_package": {}
59+
"resolved_package": {},
60+
"extra_data": {}
5861
},
5962
{
6063
"purl": "pkg:pypi/pip",
@@ -63,7 +66,8 @@
6366
"is_runtime": true,
6467
"is_optional": false,
6568
"is_resolved": false,
66-
"resolved_package": {}
69+
"resolved_package": {},
70+
"extra_data": {}
6771
},
6872
{
6973
"purl": "pkg:pypi/click",
@@ -72,7 +76,8 @@
7276
"is_runtime": true,
7377
"is_optional": false,
7478
"is_resolved": false,
75-
"resolved_package": {}
79+
"resolved_package": {},
80+
"extra_data": {}
7681
},
7782
{
7883
"purl": "pkg:pypi/intbitset",
@@ -81,7 +86,8 @@
8186
"is_runtime": true,
8287
"is_optional": false,
8388
"is_resolved": false,
84-
"resolved_package": {}
89+
"resolved_package": {},
90+
"extra_data": {}
8591
},
8692
{
8793
"purl": "pkg:pypi/requests",
@@ -90,7 +96,8 @@
9096
"is_runtime": true,
9197
"is_optional": false,
9298
"is_resolved": false,
93-
"resolved_package": {}
99+
"resolved_package": {},
100+
"extra_data": {}
94101
},
95102
{
96103
"purl": "pkg:pypi/saneyaml",
@@ -99,7 +106,8 @@
99106
"is_runtime": true,
100107
"is_optional": false,
101108
"is_resolved": false,
102-
"resolved_package": {}
109+
"resolved_package": {},
110+
"extra_data": {}
103111
},
104112
{
105113
"purl": "pkg:pypi/text-unidecode",
@@ -108,7 +116,8 @@
108116
"is_runtime": true,
109117
"is_optional": false,
110118
"is_resolved": false,
111-
"resolved_package": {}
119+
"resolved_package": {},
120+
"extra_data": {}
112121
},
113122
{
114123
"purl": "pkg:pypi/typing",
@@ -117,7 +126,8 @@
117126
"is_runtime": true,
118127
"is_optional": false,
119128
"is_resolved": false,
120-
"resolved_package": {}
129+
"resolved_package": {},
130+
"extra_data": {}
121131
},
122132
{
123133
"purl": "pkg:pypi/pytest",
@@ -126,7 +136,8 @@
126136
"is_runtime": true,
127137
"is_optional": false,
128138
"is_resolved": false,
129-
"resolved_package": {}
139+
"resolved_package": {},
140+
"extra_data": {}
130141
},
131142
{
132143
"purl": "pkg:pypi/pytest-xdist",
@@ -135,7 +146,8 @@
135146
"is_runtime": true,
136147
"is_optional": false,
137148
"is_resolved": false,
138-
"resolved_package": {}
149+
"resolved_package": {},
150+
"extra_data": {}
139151
},
140152
{
141153
"purl": "pkg:pypi/aboutcode-toolkit",
@@ -144,7 +156,8 @@
144156
"is_runtime": true,
145157
"is_optional": false,
146158
"is_resolved": false,
147-
"resolved_package": {}
159+
"resolved_package": {},
160+
"extra_data": {}
148161
},
149162
{
150163
"purl": "pkg:pypi/black",
@@ -153,7 +166,8 @@
153166
"is_runtime": true,
154167
"is_optional": false,
155168
"is_resolved": false,
156-
"resolved_package": {}
169+
"resolved_package": {},
170+
"extra_data": {}
157171
},
158172
{
159173
"purl": "pkg:pypi/sphinx",
@@ -162,7 +176,8 @@
162176
"is_runtime": true,
163177
"is_optional": false,
164178
"is_resolved": false,
165-
"resolved_package": {}
179+
"resolved_package": {},
180+
"extra_data": {}
166181
},
167182
{
168183
"purl": "pkg:pypi/sphinx-rtd-theme",
@@ -171,7 +186,8 @@
171186
"is_runtime": true,
172187
"is_optional": false,
173188
"is_resolved": false,
174-
"resolved_package": {}
189+
"resolved_package": {},
190+
"extra_data": {}
175191
},
176192
{
177193
"purl": "pkg:pypi/doc8",
@@ -180,7 +196,8 @@
180196
"is_runtime": true,
181197
"is_optional": false,
182198
"is_resolved": false,
183-
"resolved_package": {}
199+
"resolved_package": {},
200+
"extra_data": {}
184201
}
185202
],
186203
"repository_homepage_url": null,

‎tests/data/pinned-requirements.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
aboutcode-toolkit==7.0.2
12
attrs==21.4.0
23
beautifulsoup4==4.11.1
34
certifi==2022.5.18.1
@@ -11,6 +12,7 @@ importlib-metadata==4.8.3
1112
intbitset==3.0.1
1213
packageurl-python==0.9.9
1314
packaging==21.3
15+
-e git+https://github.com/nexB/python-inspector@18baae17824d6bacb4b1d519b10a0d0e50775884#egg=python_inspector
1416
pip-requirements-parser==31.2.0
1517
pkginfo2==30.0.0
1618
pyparsing==3.0.9

0 commit comments

Comments
 (0)