Skip to content

Commit bf016a2

Browse files
ziadhanyziadhany
committed
Add PyPL OSV importer
Reference: #607 Signed-off-by: Ziad <ziadhany2016@gmail.com> Add PyPL OSV Signed-off-by: Ziad <ziadhany2016@gmail.com> rename pypl_osv to pysec.py , add a test Signed-off-by: Ziad <ziadhany2016@gmail.com> squash! Add PyPI OSV importer Signed-off-by: Ziad <ziadhany2016@gmail.com> check items before accessing them , add logs Signed-off-by: Ziad <ziadhany2016@gmail.com> fix_version ranges and add logs Signed-off-by: Ziad <ziadhany2016@gmail.com> add more test , add multiple fixed_version for affected_pkg Signed-off-by: Ziad <ziadhany2016@gmail.com> add fixed_filter with a test . fix aliases , purl Signed-off-by: Ziad <ziadhany2016@gmail.com> add NotImplemented GIT-Version logs Signed-off-by: Ziad <ziadhany2016@gmail.com> add unit tests Signed-off-by: Ziad <ziadhany2016@gmail.com> add affected_pkg ranges , fix_version ecosystem support, Signed-off-by: Ziad <ziadhany2016@gmail.com>
1 parent 8c69661 commit bf016a2

5 files changed

Lines changed: 1160 additions & 1 deletion

File tree

‎requirements.txt‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,3 +20,6 @@ defusedxml==0.7.1
2020
license-expression>=21.6.14
2121

2222
Markdown==3.3.4
23+
24+
25+
dateparser~=1.1.1

‎vulnerabilities/importers/__init__.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,8 @@
2121
# Visit https://github.com/nexB/vulnerablecode/ for support and download.
2222
from vulnerabilities.importers import alpine_linux
2323
from vulnerabilities.importers import nginx
24+
from vulnerabilities.importers import pysec
2425

25-
IMPORTERS_REGISTRY = [nginx.NginxImporter, alpine_linux.AlpineImporter]
26+
IMPORTERS_REGISTRY = [nginx.NginxImporter, alpine_linux.AlpineImporter, pysec.PyPIImporter]
2627

2728
IMPORTERS_REGISTRY = {x.qualified_name: x for x in IMPORTERS_REGISTRY}

‎vulnerabilities/importers/pysec.py‎

Lines changed: 248 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,248 @@
1+
# Copyright (c) 2017 nexB Inc. and others. All rights reserved.
2+
# http://nexb.com and https://github.com/nexB/vulnerablecode/
3+
# The VulnerableCode software is licensed under the Apache License version 2.0.
4+
# Data generated with VulnerableCode require an acknowledgment.
5+
#
6+
# You may not use this software except in compliance with the License.
7+
# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
8+
# Unless required by applicable law or agreed to in writing, software distributed
9+
# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
10+
# CONDITIONS OF ANY KIND, either express or implied. See the License for the
11+
# specific language governing permissions and limitations under the License.
12+
#
13+
# When you publish or redistribute any data created with VulnerableCode or any VulnerableCode
14+
# derivative work, you must accompany this data with the following acknowledgment:
15+
#
16+
# Generated with VulnerableCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES
17+
# OR CONDITIONS OF ANY KIND, either express or implied. No content created from
18+
# VulnerableCode should be considered or used as legal advice. Consult an Attorney
19+
# for any legal advice.
20+
# VulnerableCode is a free software code scanning tool from nexB Inc. and others.
21+
# Visit https://github.com/nexB/vulnerablecode/ for support and download.
22+
import json
23+
import logging
24+
from builtins import set
25+
from io import BytesIO
26+
from typing import Iterable
27+
from zipfile import ZipFile
28+
29+
import requests
30+
from dateparser import parse
31+
from packageurl import PackageURL
32+
from univers.version_range import MavenVersionRange
33+
from univers.version_range import NugetVersionRange
34+
from univers.version_range import PypiVersionRange
35+
from univers.versions import InvalidVersion
36+
from univers.versions import MavenVersion
37+
from univers.versions import NugetVersion
38+
from univers.versions import PypiVersion
39+
from univers.versions import SemverVersion
40+
41+
from vulnerabilities.importer import AdvisoryData
42+
from vulnerabilities.importer import AffectedPackage
43+
from vulnerabilities.importer import Importer
44+
from vulnerabilities.importer import Reference
45+
from vulnerabilities.importer import VulnerabilitySeverity
46+
from vulnerabilities.severity_systems import SCORING_SYSTEMS
47+
48+
logger = logging.getLogger(__name__)
49+
50+
51+
class PyPIImporter(Importer):
52+
spdx_license_expression = "CC-BY-4.0"
53+
54+
def advisory_data(self) -> Iterable[AdvisoryData]:
55+
"""
56+
1. Fetch the data from osv api
57+
2. unzip the file
58+
3. open the file one by one
59+
4. yield the json file to parse_advisory_data
60+
"""
61+
url = "https://osv-vulnerabilities.storage.googleapis.com/PyPI/all.zip"
62+
try:
63+
response = requests.get(url).content
64+
with ZipFile(BytesIO(response)) as zip_file:
65+
for file_name in zip_file.namelist():
66+
with zip_file.open(file_name) as f:
67+
vul_info = json.loads(f.read())
68+
yield parse_advisory_data(vul_info)
69+
except requests.exceptions.RequestException:
70+
logger.error("Failed to fetch osv-vulnerabilities PyPI")
71+
72+
73+
def parse_advisory_data(raw_data: dict) -> AdvisoryData:
74+
summary = raw_data.get("summary") or ""
75+
aliases = get_aliases(raw_data)
76+
date_published = get_published_date(raw_data)
77+
severity = get_severity(raw_data)
78+
references = get_references(raw_data, severity)
79+
80+
affected_package = []
81+
if "affected" in raw_data:
82+
for affected_pkg in raw_data["affected"]:
83+
purl = get_aff_purl(affected_pkg, raw_data["id"])
84+
affected_version_range = get_aff_version_range(affected_pkg, raw_data["id"], purl)
85+
86+
for fixed_range in affected_pkg.get("ranges", []):
87+
fixed_version = get_fixed_version(fixed_range, raw_data["id"], purl)
88+
89+
for version in fixed_version:
90+
affected_package.append(
91+
AffectedPackage(
92+
package=purl,
93+
affected_version_range=affected_version_range,
94+
fixed_version=version,
95+
)
96+
)
97+
if not "affected" in raw_data:
98+
logger.error(f"affected_package not found - {raw_data['id'] !r}")
99+
100+
return AdvisoryData(
101+
aliases=aliases,
102+
summary=summary,
103+
affected_packages=affected_package,
104+
references=references,
105+
date_published=date_published,
106+
)
107+
108+
109+
def fixed_filter(fixed_range) -> []:
110+
"""
111+
>>> fixed_filter({"type": "SEMVER", "events": [{"introduced": "0"}, {"fixed": "1.6.0"}]})
112+
["1.6.0"]
113+
>>> fixed_filter({"type": "ECOSYSTEM","events":[{"introduced": "0"},{"fixed": "1.0.0"},{"fixed": "9.0.0"}]})
114+
["1.0.0", "9.0.0"]
115+
"""
116+
filter_fixed = list(filter(lambda x: x.keys() == {"fixed"}, fixed_range["events"]))
117+
list_fixed = [i["fixed"] for i in filter_fixed]
118+
return list_fixed
119+
120+
121+
def get_aliases(raw_data) -> []:
122+
"""
123+
aliases field is optional , id is required and these are all aliases from our perspective
124+
converting list of two fields to a dict then , convert it to a list to make sure a list is unique
125+
"""
126+
vulnerability_id = raw_data.get("id")
127+
vulnerability_aliases = raw_data.get("aliases") or []
128+
if vulnerability_id:
129+
vulnerability_aliases.append(vulnerability_id)
130+
return list(set(vulnerability_aliases))
131+
132+
133+
def get_published_date(raw_data):
134+
if "published" in raw_data:
135+
return parse(raw_data["published"])
136+
else:
137+
logger.warning(f"date_published not found {raw_data['id'] !r}")
138+
139+
140+
def get_severity(raw_data) -> []:
141+
severity = []
142+
if "severity" in raw_data:
143+
for sever_list in raw_data["severity"]:
144+
if "type" in sever_list and sever_list["type"] == "CVSS_V3":
145+
severity.append(
146+
VulnerabilitySeverity(
147+
system=SCORING_SYSTEMS["cvssv3_vector"],
148+
value=sever_list["score"],
149+
)
150+
)
151+
if "ecosystem_specific" in raw_data and "severity" in raw_data["ecosystem_specific"]:
152+
severity.append(
153+
VulnerabilitySeverity(
154+
system=SCORING_SYSTEMS["generic_textual"],
155+
value=raw_data["ecosystem_specific"]["severity"],
156+
)
157+
)
158+
else:
159+
logger.warning(f"severity not found- {raw_data['id']!r}")
160+
161+
return severity
162+
163+
164+
def get_references(raw_data, severity) -> []:
165+
if "references" in raw_data:
166+
return [
167+
Reference(url=ref["url"], severities=severity) for ref in raw_data["references"] if ref
168+
]
169+
else:
170+
return []
171+
172+
173+
def get_aff_purl(affected_pkg, raw_id):
174+
package = affected_pkg["package"]
175+
if "purl" in package:
176+
try:
177+
return PackageURL.from_string(package["purl"])
178+
except ValueError:
179+
logger.error(f"PackageURL ValueError - {raw_id !r} - purl: {package['purl'] !r}")
180+
181+
if "ecosystem" in package and "name" in package:
182+
return PackageURL(type=package["ecosystem"], name=package["name"])
183+
else:
184+
logger.error(f"purl affected_pkg not found - {raw_id !r}")
185+
186+
187+
def get_aff_version_range(affected_pkg, raw_id, purl):
188+
if "versions" in affected_pkg and len(affected_pkg["versions"]) < 200:
189+
if purl.type == "pypi":
190+
try:
191+
return PypiVersionRange(affected_pkg["versions"])
192+
except Exception as e:
193+
logger.error(f"affected_pkg_version_range Error - {raw_id !r} - purl: {e !r}")
194+
195+
elif purl.type == "maven":
196+
try:
197+
return MavenVersionRange(affected_pkg["versions"])
198+
except Exception as e:
199+
logger.error(f"affected_pkg_version_range Error - {raw_id !r} - purl: {e !r}")
200+
201+
elif purl.type == "nuget":
202+
try:
203+
NugetVersionRange(affected_pkg["versions"])
204+
except Exception as e:
205+
logger.error(f"Invalid Version - NugetVersion - {raw_id !r} - {e !r}")
206+
207+
else:
208+
logger.error(f"NotImplementedError Ecosystem fixed range - {raw_id !r} ")
209+
210+
211+
def get_fixed_version(fixed_range, raw_id, purl) -> {}:
212+
fixed_version = set()
213+
purl_type = purl.type
214+
if "type" in fixed_range:
215+
list_fixed = fixed_filter(fixed_range)
216+
for i in list_fixed:
217+
if fixed_range["type"] == "ECOSYSTEM":
218+
if purl_type == "pypi":
219+
try:
220+
fixed_version.add(PypiVersion(i))
221+
except InvalidVersion:
222+
logger.error(f"Invalid Version - PypiVersion - {raw_id !r} - {i !r}")
223+
elif purl_type == "maven":
224+
try:
225+
fixed_version.add(MavenVersion(i))
226+
except InvalidVersion:
227+
logger.error(f"Invalid Version - MavenVersion - {raw_id !r} - {i !r}")
228+
elif purl_type == "nuget":
229+
try:
230+
fixed_version.add(NugetVersion(i))
231+
except InvalidVersion:
232+
logger.error(f"Invalid Version - NugetVersion - {raw_id !r} - {i !r}")
233+
234+
else:
235+
logger.error(f"NotImplementedError Ecosystem aff range - {raw_id !r} - {i !r}")
236+
if fixed_range["type"] == "SEMVER":
237+
try:
238+
fixed_version.add(SemverVersion(i))
239+
except InvalidVersion:
240+
logger.error(f"Invalid Version - SemverVersion - {raw_id !r} - {i !r}")
241+
if fixed_range["type"] == "GIT":
242+
# TODO add GitHubVersion univers fix_version
243+
logger.error(f"NotImplementedError GIT Version - {raw_id !r} - {i !r}")
244+
245+
else:
246+
logger.error(f"Invalid type - {raw_id!r}")
247+
248+
return fixed_version

0 commit comments

Comments
 (0)