Skip to content

Commit eae09e7

Browse files
ziadhanyziadhany
committed
Add PyPL OSV importer
Reference: #607 Signed-off-by: Ziad <ziadhany2016@gmail.com> Add PyPL OSV Signed-off-by: Ziad <ziadhany2016@gmail.com> rename pypl_osv to pysec.py , add a test Signed-off-by: Ziad <ziadhany2016@gmail.com> squash! Add PyPI OSV importer Signed-off-by: Ziad <ziadhany2016@gmail.com> check items before accessing them , add logs Signed-off-by: Ziad <ziadhany2016@gmail.com> fix_version ranges and add logs Signed-off-by: Ziad <ziadhany2016@gmail.com> add more test , add multiple fixed_version for affected_pkg Signed-off-by: Ziad <ziadhany2016@gmail.com>
1 parent 8c69661 commit eae09e7

4 files changed

Lines changed: 826 additions & 1 deletion

File tree

‎vulnerabilities/importers/__init__.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,8 @@
2121
# Visit https://github.com/nexB/vulnerablecode/ for support and download.
2222
from vulnerabilities.importers import alpine_linux
2323
from vulnerabilities.importers import nginx
24+
from vulnerabilities.importers import pysec
2425

25-
IMPORTERS_REGISTRY = [nginx.NginxImporter, alpine_linux.AlpineImporter]
26+
IMPORTERS_REGISTRY = [nginx.NginxImporter, alpine_linux.AlpineImporter, pysec.PyPIImporter]
2627

2728
IMPORTERS_REGISTRY = {x.qualified_name: x for x in IMPORTERS_REGISTRY}

‎vulnerabilities/importers/pysec.py‎

Lines changed: 178 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,178 @@
1+
# Copyright (c) 2017 nexB Inc. and others. All rights reserved.
2+
# http://nexb.com and https://github.com/nexB/vulnerablecode/
3+
# The VulnerableCode software is licensed under the Apache License version 2.0.
4+
# Data generated with VulnerableCode require an acknowledgment.
5+
#
6+
# You may not use this software except in compliance with the License.
7+
# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
8+
# Unless required by applicable law or agreed to in writing, software distributed
9+
# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
10+
# CONDITIONS OF ANY KIND, either express or implied. See the License for the
11+
# specific language governing permissions and limitations under the License.
12+
#
13+
# When you publish or redistribute any data created with VulnerableCode or any VulnerableCode
14+
# derivative work, you must accompany this data with the following acknowledgment:
15+
#
16+
# Generated with VulnerableCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES
17+
# OR CONDITIONS OF ANY KIND, either express or implied. No content created from
18+
# VulnerableCode should be considered or used as legal advice. Consult an Attorney
19+
# for any legal advice.
20+
# VulnerableCode is a free software code scanning tool from nexB Inc. and others.
21+
# Visit https://github.com/nexB/vulnerablecode/ for support and download.
22+
import json
23+
import logging
24+
from datetime import datetime
25+
from datetime import timezone
26+
from io import BytesIO
27+
from typing import Iterable
28+
from zipfile import ZipFile
29+
30+
import requests
31+
from packageurl import PackageURL
32+
from univers.version_range import PypiVersionRange
33+
from univers.versions import InvalidVersion
34+
from univers.versions import PypiVersion
35+
from univers.versions import SemverVersion
36+
37+
from vulnerabilities.importer import AdvisoryData
38+
from vulnerabilities.importer import AffectedPackage
39+
from vulnerabilities.importer import Importer
40+
from vulnerabilities.importer import Reference
41+
from vulnerabilities.importer import VulnerabilitySeverity
42+
from vulnerabilities.severity_systems import SCORING_SYSTEMS
43+
44+
logger = logging.getLogger(__name__)
45+
46+
47+
class PyPIImporter(Importer):
48+
spdx_license_expression = "Apache-2.0"
49+
50+
def advisory_data(self) -> Iterable[AdvisoryData]:
51+
"""
52+
1. Fetch the data from osv api
53+
2. unzip the file
54+
3. open the file one by one
55+
4. yield the json file to parse_advisory_data
56+
"""
57+
url = "https://osv-vulnerabilities.storage.googleapis.com/PyPI/all.zip"
58+
response = requests.get(url).content
59+
try:
60+
with ZipFile(BytesIO(response)) as zip_file:
61+
for file_name in zip_file.namelist():
62+
with zip_file.open(file_name) as f:
63+
vul_info = json.loads(f.read())
64+
yield parse_advisory_data(vul_info)
65+
except requests.exceptions.RequestException:
66+
logger.error(f"Failed to fetch osv-vulnerabilities PyPI: HTTP ")
67+
68+
69+
def parse_advisory_data(raw_data: dict) -> AdvisoryData:
70+
if "summary" in raw_data:
71+
summary = raw_data["summary"]
72+
else:
73+
summary = ""
74+
75+
aliases = raw_data["aliases"] if "aliases" in raw_data else raw_data["id"]
76+
77+
if "published" in raw_data:
78+
date_published = datetime.strptime(
79+
raw_data["published"][0:19], "%Y-%m-%dT%H:%M:%S"
80+
).replace(tzinfo=timezone.utc)
81+
else:
82+
date_published = None
83+
logger.error("date_published not found " + raw_data["id"])
84+
85+
severity = []
86+
if "severity" in raw_data:
87+
for sever_list in raw_data["severity"]:
88+
if "type" in sever_list and sever_list["type"] == "CVSS_V3":
89+
severity.append(
90+
VulnerabilitySeverity(
91+
system=SCORING_SYSTEMS["cvssv3_vector"],
92+
value=sever_list["score"],
93+
)
94+
)
95+
elif "ecosystem_specific" in raw_data and "severity" in raw_data["ecosystem_specific"]:
96+
severity.append(
97+
VulnerabilitySeverity(
98+
system=SCORING_SYSTEMS["generic_textual"],
99+
value=raw_data["ecosystem_specific"]["severity"],
100+
)
101+
)
102+
else:
103+
severity = []
104+
logger.warning(f"severity not found- {raw_data['id']!r}")
105+
106+
references = []
107+
if "references" in raw_data:
108+
references = [
109+
Reference(url=ref["url"], severities=severity) for ref in raw_data["references"] if ref
110+
]
111+
112+
affected_package = []
113+
if "affected" in raw_data:
114+
for affected_pkg in raw_data["affected"]:
115+
package = affected_pkg["package"]
116+
if "ecosystem" in package and "name" in package:
117+
purl = PackageURL(type=package["ecosystem"], name=package["name"])
118+
else:
119+
purl = ""
120+
logger.error(f"purl affected_pkg not found - {raw_data['id'] !r}")
121+
122+
affected_version_range = None
123+
if "versions" in affected_pkg:
124+
affected_version_range = (
125+
# HINT: Values larger than 1/3 of a buffer page cannot be indexed.
126+
PypiVersionRange(affected_pkg["versions"][slice(200)])
127+
)
128+
129+
if "ranges" in affected_pkg:
130+
for fix_range in affected_pkg["ranges"]:
131+
fixed_version = set()
132+
if "type" in fix_range:
133+
if fix_range["type"] == "ECOSYSTEM":
134+
filter_fixed = list(
135+
filter(lambda x: x.keys() == {"fixed"}, fix_range["events"])
136+
)
137+
list_fixed = [i["fixed"] for i in filter_fixed]
138+
for i in list_fixed:
139+
try:
140+
fixed_version.add(PypiVersion(i))
141+
except InvalidVersion:
142+
logger.error(
143+
f"fixed_version InvalidVersion - PypiVersion - {raw_data['id'] !r}"
144+
)
145+
146+
if fix_range["type"] == "SEMVER":
147+
filter_fixed = list(
148+
filter(lambda x: x.keys() == {"fixed"}, fix_range["events"])
149+
)
150+
list_fixed = [i["fixed"] for i in filter_fixed]
151+
152+
for i in list_fixed:
153+
try:
154+
fixed_version.add(SemverVersion(i))
155+
except InvalidVersion:
156+
logger.error(
157+
f"fixed_version InvalidVersion - SemverVersion - {raw_data['id'] !r}"
158+
)
159+
160+
# TODO add GitHubVersion univers fix_version
161+
for version in fixed_version:
162+
affected_package.append(
163+
AffectedPackage(
164+
package=purl,
165+
affected_version_range=affected_version_range,
166+
fixed_version=version,
167+
)
168+
)
169+
else:
170+
logger.error(f"affected_package not found - {raw_data['id'] !r}")
171+
172+
return AdvisoryData(
173+
aliases=aliases,
174+
summary=summary,
175+
affected_packages=affected_package,
176+
references=references,
177+
date_published=date_published,
178+
)

0 commit comments

Comments
 (0)