Skip to content

Commit 065bdce

Browse files
TG1999TG1999
committed
Address review comments
Signed-off-by: Tushar Goel <tushar.goel.dav@gmail.com>
1 parent 2b8134a commit 065bdce

3 files changed

Lines changed: 44 additions & 88 deletions

File tree

‎src/_packagedcode/pypi.py‎

Lines changed: 25 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -9,15 +9,16 @@
99
#
1010

1111
import ast
12+
from configparser import ConfigParser
1213
import json
1314
import logging
15+
from pathlib import Path
1416
import os
1517
import re
1618
import sys
1719
from typing import NamedTuple
20+
import tempfile
1821
import zipfile
19-
from configparser import ConfigParser
20-
from pathlib import Path
2122

2223
import dparse2
2324
import packaging
@@ -456,7 +457,7 @@ def parse_metadata(location, datasource_id, package_type):
456457
primary_language='Python',
457458
name=name,
458459
version=version,
459-
description=get_description( metainfo= meta, location= str(location)),
460+
description=get_description(metainfo=meta, location=str(location)),
460461
#TODO: https://github.com/nexB/scancode-toolkit/issues/3014
461462
declared_license=get_declared_license(meta),
462463
keywords=get_keywords(meta),
@@ -707,27 +708,26 @@ def parse(cls, location):
707708
"setup_requires": "setup",
708709
"python_requires": "python",
709710
}
710-
for sub_section in scope_by_sub_section:
711+
for sub_section, scope in scope_by_sub_section.items():
711712
if sub_section not in section:
712713
continue
713-
scope = scope_by_sub_section[sub_section]
714714
if scope != "python":
715715
reqs = list(get_requirement_from_section(section=section, sub_section=sub_section))
716716
dependent_packages.extend(cls.parse_reqs(reqs, scope))
717717
continue
718-
python_requires = section[sub_section]
718+
python_requires_specifier = section[sub_section]
719719
purl = PackageURL(
720+
type="generic",
720721
name="python",
721-
type="generic"
722722
)
723-
resolved_purl = is_purl_resolved(purl = purl, specifiers= SpecifierSet(python_requires))
723+
resolved_purl = get_resolved_purl(purl=purl, specifiers=SpecifierSet(python_requires_specifier))
724724
dependent_packages.append(models.DependentPackage(
725725
purl=str(resolved_purl.purl),
726726
scope=scope,
727727
is_runtime=True,
728728
is_optional=False,
729729
is_resolved=resolved_purl.is_resolved,
730-
extracted_requirement=f"python_requires{python_requires}",
730+
extracted_requirement=f"python_requires{python_requires_specifier}",
731731
))
732732

733733
if section.name == "options.extras_require":
@@ -784,7 +784,7 @@ def parse_reqs(cls, reqs, scope):
784784
name = canonicalize_name(req_parsed.name)
785785
purl = PackageURL(type="pypi", name=name)
786786
specifiers = req_parsed.specifier._specs
787-
resolved_purl = is_purl_resolved(purl = purl, specifiers= specifiers)
787+
resolved_purl = get_resolved_purl(purl=purl, specifiers=specifiers)
788788
dependent_packages.append(
789789
models.DependentPackage(
790790
purl=str(resolved_purl.purl),
@@ -798,9 +798,10 @@ def parse_reqs(cls, reqs, scope):
798798
return dependent_packages
799799

800800

801-
def is_purl_resolved(purl: PackageURL, specifiers: SpecifierSet):
801+
def get_resolved_purl(purl: PackageURL, specifiers: SpecifierSet):
802802
"""
803-
Check if the purl is resolved
803+
Check if the purl is resolved and return a ResolvedPurl.
804+
If the purl is resolved, update its version to the pinned version
804805
"""
805806
is_resolved = False
806807
if len(specifiers) == 1:
@@ -901,7 +902,7 @@ def get_requirements_txt_dependencies(location, include_nested=False):
901902
include_nested=include_nested,
902903
)
903904
if not req_file or not req_file.requirements:
904-
return []
905+
return [], {}
905906

906907
# for now we ignore errors
907908
extra_data = {}
@@ -1979,23 +1980,14 @@ def compute_normalized_license(declared_license):
19791980

19801981
def get_requirement_from_section(section, sub_section):
19811982
"""
1982-
Generate requirements from the `sub_section`
1983-
"""
1984-
content = section.get(sub_section) or ""
1985-
for req in content.splitlines():
1986-
if not req:
1987-
continue
1988-
#pytest-mypy >= 0.9.1; \
1989-
req = req.replace("; \\", "")
1990-
# pip>=19.1 # For proper file:// URLs support.
1991-
if "#" in req:
1992-
req , _ = req.rsplit("#")
1993-
try:
1994-
Requirement(req)
1995-
yield req
1996-
except:
1997-
#pure-eval; black; tox
1998-
req_split_by_semi_colon = req.split(";")
1999-
req_split_by_semi_colon = [req.strip() for req in req_split_by_semi_colon if req]
2000-
for req in req_split_by_semi_colon:
2001-
yield req
1983+
Yield extracted requirement from the ``sub_section`` key of of a ``section``
1984+
mapping (from a setup.cfg)
1985+
"""
1986+
content = section.get(sub_section, "")
1987+
temp = tempfile.NamedTemporaryFile(delete=False)
1988+
location = temp.name
1989+
with open(location, "w") as f:
1990+
f.write(content)
1991+
packages, _ = get_requirements_txt_dependencies(location=location)
1992+
for req in packages:
1993+
yield req.extracted_requirement

‎tests/data/parse-reqs.json‎

Lines changed: 17 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -30,17 +30,8 @@
3030
"extra_data": {},
3131
"dependencies": [
3232
{
33-
"purl": "pkg:pypi/pytest-mypy",
34-
"extracted_requirement": "pytest-mypy >= 0.9.1",
35-
"scope": "install",
36-
"is_runtime": true,
37-
"is_optional": false,
38-
"is_resolved": false,
39-
"resolved_package": {}
40-
},
41-
{
42-
"purl": "pkg:pypi/pytest-cov",
43-
"extracted_requirement": "pytest-cov >= 2.5.1",
33+
"purl": "pkg:pypi/pytest-black",
34+
"extracted_requirement": "pytest-black>=0.3.7; platform_python_implementation != \"PyPy\"",
4435
"scope": "install",
4536
"is_runtime": true,
4637
"is_optional": false,
@@ -49,7 +40,7 @@
4940
},
5041
{
5142
"purl": "pkg:pypi/attrs",
52-
"extracted_requirement": "attrs >= 18.1, !=20.1.0",
43+
"extracted_requirement": "attrs>=18.1,!=20.1.0",
5344
"scope": "install",
5445
"is_runtime": true,
5546
"is_optional": false,
@@ -58,7 +49,7 @@
5849
},
5950
{
6051
"purl": "pkg:pypi/beautifulsoup4",
61-
"extracted_requirement": "Beautifulsoup4 >= 4.0.0",
52+
"extracted_requirement": "Beautifulsoup4>=4.0.0",
6253
"scope": "install",
6354
"is_runtime": true,
6455
"is_optional": false,
@@ -67,7 +58,7 @@
6758
},
6859
{
6960
"purl": "pkg:pypi/pip",
70-
"extracted_requirement": "pip>=19.1 ",
61+
"extracted_requirement": "pip>=19.1",
7162
"scope": "install",
7263
"is_runtime": true,
7364
"is_optional": false,
@@ -76,7 +67,7 @@
7667
},
7768
{
7869
"purl": "pkg:pypi/click",
79-
"extracted_requirement": "click >= 6.7, !=7.0",
70+
"extracted_requirement": "click>=6.7,!=7.0",
8071
"scope": "install",
8172
"is_runtime": true,
8273
"is_optional": false,
@@ -85,7 +76,7 @@
8576
},
8677
{
8778
"purl": "pkg:pypi/intbitset",
88-
"extracted_requirement": "intbitset >= 2.3.0",
79+
"extracted_requirement": "intbitset>=2.3.0",
8980
"scope": "install",
9081
"is_runtime": true,
9182
"is_optional": false,
@@ -94,7 +85,7 @@
9485
},
9586
{
9687
"purl": "pkg:pypi/requests",
97-
"extracted_requirement": "requests >= 2.7.0",
88+
"extracted_requirement": "requests>=2.7.0",
9889
"scope": "install",
9990
"is_runtime": true,
10091
"is_optional": false,
@@ -103,7 +94,7 @@
10394
},
10495
{
10596
"purl": "pkg:pypi/saneyaml",
106-
"extracted_requirement": "saneyaml >= 0.5.2",
97+
"extracted_requirement": "saneyaml>=0.5.2",
10798
"scope": "install",
10899
"is_runtime": true,
109100
"is_optional": false,
@@ -112,34 +103,7 @@
112103
},
113104
{
114105
"purl": "pkg:pypi/text-unidecode",
115-
"extracted_requirement": "text_unidecode >= 1.0",
116-
"scope": "install",
117-
"is_runtime": true,
118-
"is_optional": false,
119-
"is_resolved": false,
120-
"resolved_package": {}
121-
},
122-
{
123-
"purl": "pkg:pypi/pure-eval",
124-
"extracted_requirement": "pure-eval",
125-
"scope": "install",
126-
"is_runtime": true,
127-
"is_optional": false,
128-
"is_resolved": false,
129-
"resolved_package": {}
130-
},
131-
{
132-
"purl": "pkg:pypi/black",
133-
"extracted_requirement": "black",
134-
"scope": "install",
135-
"is_runtime": true,
136-
"is_optional": false,
137-
"is_resolved": false,
138-
"resolved_package": {}
139-
},
140-
{
141-
"purl": "pkg:pypi/tox",
142-
"extracted_requirement": "tox",
106+
"extracted_requirement": "text_unidecode>=1.0",
143107
"scope": "install",
144108
"is_runtime": true,
145109
"is_optional": false,
@@ -148,7 +112,7 @@
148112
},
149113
{
150114
"purl": "pkg:pypi/typing",
151-
"extracted_requirement": "typing >=3.6, < 3.7; python_version < \"3.7\"",
115+
"extracted_requirement": "typing>=3.6,<3.7; python_version < \"3.7\"",
152116
"scope": "install",
153117
"is_runtime": true,
154118
"is_optional": false,
@@ -157,7 +121,7 @@
157121
},
158122
{
159123
"purl": "pkg:pypi/pytest",
160-
"extracted_requirement": "pytest >= 6, != 7.0.0",
124+
"extracted_requirement": "pytest>=6,!=7.0.0",
161125
"scope": "testing",
162126
"is_runtime": true,
163127
"is_optional": false,
@@ -166,7 +130,7 @@
166130
},
167131
{
168132
"purl": "pkg:pypi/pytest-xdist",
169-
"extracted_requirement": "pytest-xdist >= 2",
133+
"extracted_requirement": "pytest-xdist>=2",
170134
"scope": "testing",
171135
"is_runtime": true,
172136
"is_optional": false,
@@ -175,7 +139,7 @@
175139
},
176140
{
177141
"purl": "pkg:pypi/aboutcode-toolkit",
178-
"extracted_requirement": "aboutcode-toolkit >= 6.0.0",
142+
"extracted_requirement": "aboutcode-toolkit>=6.0.0",
179143
"scope": "testing",
180144
"is_runtime": true,
181145
"is_optional": false,
@@ -193,7 +157,7 @@
193157
},
194158
{
195159
"purl": "pkg:pypi/sphinx",
196-
"extracted_requirement": "Sphinx >= 3.3.1",
160+
"extracted_requirement": "Sphinx>=3.3.1",
197161
"scope": "docs",
198162
"is_runtime": true,
199163
"is_optional": false,
@@ -202,7 +166,7 @@
202166
},
203167
{
204168
"purl": "pkg:pypi/sphinx-rtd-theme",
205-
"extracted_requirement": "sphinx-rtd-theme >= 0.5.0",
169+
"extracted_requirement": "sphinx-rtd-theme>=0.5.0",
206170
"scope": "docs",
207171
"is_runtime": true,
208172
"is_optional": false,
@@ -211,7 +175,7 @@
211175
},
212176
{
213177
"purl": "pkg:pypi/doc8",
214-
"extracted_requirement": "doc8 >= 0.8.1",
178+
"extracted_requirement": "doc8>=0.8.1",
215179
"scope": "docs",
216180
"is_runtime": true,
217181
"is_optional": false,

‎tests/data/setup.cfg‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
[options]
22
install_requires =
3-
pytest-mypy >= 0.9.1; \
4-
pytest-cov >= 2.5.1; \
3+
pytest-black >= 0.3.7; \
4+
python_implementation != "PyPy"
55
attrs >= 18.1, !=20.1.0
66
Beautifulsoup4 >= 4.0.0
77
pip>=19.1 # For proper file:// URLs support

0 commit comments

Comments
 (0)