Skip to content

Commit 006b85b

Browse files
ziadhanyziadhany
committed
Add PyPL OSV importer
Reference: #607 Signed-off-by: Ziad <ziadhany2016@gmail.com> Add PyPL OSV Signed-off-by: Ziad <ziadhany2016@gmail.com> rename pypl_osv to pysec.py , add a test Signed-off-by: Ziad <ziadhany2016@gmail.com> squash! Add PyPI OSV importer Signed-off-by: Ziad <ziadhany2016@gmail.com> check items before accessing them , add logs Signed-off-by: Ziad <ziadhany2016@gmail.com>
1 parent 8c69661 commit 006b85b

4 files changed

Lines changed: 571 additions & 1 deletion

File tree

‎vulnerabilities/importers/__init__.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,8 @@
2121
# Visit https://github.com/nexB/vulnerablecode/ for support and download.
2222
from vulnerabilities.importers import alpine_linux
2323
from vulnerabilities.importers import nginx
24+
from vulnerabilities.importers import pysec
2425

25-
IMPORTERS_REGISTRY = [nginx.NginxImporter, alpine_linux.AlpineImporter]
26+
IMPORTERS_REGISTRY = [nginx.NginxImporter, alpine_linux.AlpineImporter, pysec.PyPlImporter]
2627

2728
IMPORTERS_REGISTRY = {x.qualified_name: x for x in IMPORTERS_REGISTRY}

‎vulnerabilities/importers/pysec.py‎

Lines changed: 174 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,174 @@
1+
# Copyright (c) 2017 nexB Inc. and others. All rights reserved.
2+
# http://nexb.com and https://github.com/nexB/vulnerablecode/
3+
# The VulnerableCode software is licensed under the Apache License version 2.0.
4+
# Data generated with VulnerableCode require an acknowledgment.
5+
#
6+
# You may not use this software except in compliance with the License.
7+
# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
8+
# Unless required by applicable law or agreed to in writing, software distributed
9+
# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
10+
# CONDITIONS OF ANY KIND, either express or implied. See the License for the
11+
# specific language governing permissions and limitations under the License.
12+
#
13+
# When you publish or redistribute any data created with VulnerableCode or any VulnerableCode
14+
# derivative work, you must accompany this data with the following acknowledgment:
15+
#
16+
# Generated with VulnerableCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES
17+
# OR CONDITIONS OF ANY KIND, either express or implied. No content created from
18+
# VulnerableCode should be considered or used as legal advice. Consult an Attorney
19+
# for any legal advice.
20+
# VulnerableCode is a free software code scanning tool from nexB Inc. and others.
21+
# Visit https://github.com/nexB/vulnerablecode/ for support and download.
22+
import json
23+
import logging
24+
from datetime import datetime
25+
from datetime import timezone
26+
from io import BytesIO
27+
from typing import Iterable
28+
from zipfile import ZipFile
29+
30+
import requests
31+
from packageurl import PackageURL
32+
from univers.version_range import GitHubVersionRange
33+
from univers.version_range import PypiVersionRange
34+
from univers.versions import SemverVersion
35+
36+
from vulnerabilities.importer import AdvisoryData
37+
from vulnerabilities.importer import AffectedPackage
38+
from vulnerabilities.importer import Importer
39+
from vulnerabilities.importer import Reference
40+
from vulnerabilities.importer import VulnerabilitySeverity
41+
from vulnerabilities.severity_systems import SCORING_SYSTEMS
42+
43+
logger = logging.getLogger(__name__)
44+
45+
46+
class PyPlImporter(Importer):
47+
spdx_license_expression = "Apache-2.0"
48+
49+
def advisory_data(self) -> Iterable[AdvisoryData]:
50+
url = "https://osv-vulnerabilities.storage.googleapis.com/PyPI/all.zip"
51+
response = requests.get(url).content
52+
try:
53+
with ZipFile(BytesIO(response)) as zip_file:
54+
for file_name in zip_file.namelist():
55+
with zip_file.open(file_name) as f:
56+
vul_info = json.loads(f.read())
57+
yield parse_advisory_data(vul_info)
58+
except requests.exceptions.RequestException:
59+
logger.error(f"Failed to fetch osv-vulnerabilities PyPI: HTTP ")
60+
61+
62+
def parse_advisory_data(raw_data: dict) -> AdvisoryData:
63+
64+
if "summary" in raw_data:
65+
summary = raw_data["summary"]
66+
elif "details" in raw_data:
67+
summary = raw_data["details"][slice(50)]
68+
else:
69+
summary = ""
70+
logger.error(f"summary not found- {raw_data['id'] !r}")
71+
72+
aliases = raw_data["aliases"] if "aliases" in raw_data else raw_data["id"]
73+
74+
if raw_data["published"] is not None:
75+
date_published = datetime.strptime(
76+
raw_data["published"][0:19], "%Y-%m-%dT%H:%M:%S"
77+
).replace(tzinfo=timezone.utc)
78+
else:
79+
date_published = None
80+
logger.error("date_published not found " + raw_data["id"])
81+
82+
severity = []
83+
if "severity" in raw_data:
84+
for sever_list in raw_data["severity"]:
85+
if "type" in sever_list and sever_list["type"] == "CVSS_V3":
86+
severity.append(
87+
VulnerabilitySeverity(
88+
system=SCORING_SYSTEMS["cvssv3_vector"],
89+
value=sever_list["score"],
90+
)
91+
)
92+
elif "ecosystem_specific" in raw_data and "severity" in raw_data["ecosystem_specific"]:
93+
severity.append(
94+
VulnerabilitySeverity(
95+
system=SCORING_SYSTEMS["generic_textual"],
96+
value=raw_data["ecosystem_specific"]["severity"],
97+
)
98+
)
99+
else:
100+
severity = []
101+
logger.error(f"severity not found- {raw_data['id'] !r}")
102+
103+
references = []
104+
if "references" in raw_data:
105+
for ref in raw_data["references"]:
106+
if ref["url"] is None:
107+
continue
108+
else:
109+
references.append(Reference(url=ref["url"], severities=severity))
110+
else:
111+
logger.error(f"references not found - {raw_data['id'] !r}")
112+
113+
affected_package = []
114+
if "affected" in raw_data:
115+
for affected_pkg in raw_data["affected"]:
116+
if (affected_pkg["package"]["ecosystem"] is not None) and (
117+
affected_pkg["package"]["name"] is not None
118+
):
119+
purl = PackageURL(
120+
type=affected_pkg["package"]["ecosystem"], name=affected_pkg["package"]["name"]
121+
)
122+
else:
123+
purl = ""
124+
logger.error(f"purl affected_pkg not found - {raw_data['id'] !r}")
125+
126+
affected_version_range = (
127+
PypiVersionRange(affected_pkg["versions"][slice(50)])
128+
if "versions" in affected_pkg
129+
else None
130+
)
131+
132+
if "ranges" in affected_pkg:
133+
for fix_range in affected_pkg["ranges"]:
134+
fixed_version = set()
135+
if "type" in fix_range:
136+
if fix_range["type"] == "ECOSYSTEM":
137+
filter_fixed = list(
138+
filter(lambda x: x.keys() == {"fixed"}, fix_range["events"])
139+
)
140+
list_fixed = [i["fixed"] for i in filter_fixed]
141+
fixed_version.add(PypiVersionRange(list_fixed))
142+
143+
if fix_range["type"] == "SEMVER":
144+
filter_fixed = list(
145+
filter(lambda x: x.keys() == {"fixed"}, fix_range["events"])
146+
)
147+
list_fixed = [i["fixed"] for i in filter_fixed]
148+
for i in list_fixed:
149+
fixed_version.add(SemverVersion(i))
150+
151+
if fix_range["type"] == "GIT":
152+
filter_fixed = list(
153+
filter(lambda x: x.keys() == {"fixed"}, fix_range["events"])
154+
)
155+
list_fixed = [i["fixed"] for i in filter_fixed]
156+
fixed_version.add(GitHubVersionRange(list_fixed))
157+
158+
affected_package.append(
159+
AffectedPackage(
160+
package=purl,
161+
affected_version_range=affected_version_range,
162+
fixed_version=fixed_version,
163+
)
164+
)
165+
else:
166+
logger.error(f"affected_package not found - {raw_data['id'] !r}")
167+
168+
return AdvisoryData(
169+
aliases=aliases,
170+
summary=summary,
171+
affected_packages=affected_package,
172+
references=references,
173+
date_published=date_published,
174+
)

0 commit comments

Comments
 (0)