Skip to content

Commit 4e784a1

Browse files
ziadhanyziadhany
committed
Add PyPL OSV importer
Reference: #607 Signed-off-by: Ziad <ziadhany2016@gmail.com> Add PyPL OSV Signed-off-by: Ziad <ziadhany2016@gmail.com> rename pypl_osv to pysec.py , add a test Signed-off-by: Ziad <ziadhany2016@gmail.com> squash! Add PyPI OSV importer Signed-off-by: Ziad <ziadhany2016@gmail.com> check items before accessing them , add logs Signed-off-by: Ziad <ziadhany2016@gmail.com> fix_version ranges and add logs Signed-off-by: Ziad <ziadhany2016@gmail.com> add more test , add multiple fixed_version for affected_pkg Signed-off-by: Ziad <ziadhany2016@gmail.com> add fixed_filter with a test . fix aliases , purl Signed-off-by: Ziad <ziadhany2016@gmail.com> add NotImplemented GIT-Version logs Signed-off-by: Ziad <ziadhany2016@gmail.com> add unit tests Signed-off-by: Ziad <ziadhany2016@gmail.com>
1 parent 8c69661 commit 4e784a1

4 files changed

Lines changed: 1063 additions & 1 deletion

File tree

‎vulnerabilities/importers/__init__.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,8 @@
2121
# Visit https://github.com/nexB/vulnerablecode/ for support and download.
2222
from vulnerabilities.importers import alpine_linux
2323
from vulnerabilities.importers import nginx
24+
from vulnerabilities.importers import pysec
2425

25-
IMPORTERS_REGISTRY = [nginx.NginxImporter, alpine_linux.AlpineImporter]
26+
IMPORTERS_REGISTRY = [nginx.NginxImporter, alpine_linux.AlpineImporter, pysec.PyPIImporter]
2627

2728
IMPORTERS_REGISTRY = {x.qualified_name: x for x in IMPORTERS_REGISTRY}

‎vulnerabilities/importers/pysec.py‎

Lines changed: 206 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,206 @@
1+
# Copyright (c) 2017 nexB Inc. and others. All rights reserved.
2+
# http://nexb.com and https://github.com/nexB/vulnerablecode/
3+
# The VulnerableCode software is licensed under the Apache License version 2.0.
4+
# Data generated with VulnerableCode require an acknowledgment.
5+
#
6+
# You may not use this software except in compliance with the License.
7+
# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
8+
# Unless required by applicable law or agreed to in writing, software distributed
9+
# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
10+
# CONDITIONS OF ANY KIND, either express or implied. See the License for the
11+
# specific language governing permissions and limitations under the License.
12+
#
13+
# When you publish or redistribute any data created with VulnerableCode or any VulnerableCode
14+
# derivative work, you must accompany this data with the following acknowledgment:
15+
#
16+
# Generated with VulnerableCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES
17+
# OR CONDITIONS OF ANY KIND, either express or implied. No content created from
18+
# VulnerableCode should be considered or used as legal advice. Consult an Attorney
19+
# for any legal advice.
20+
# VulnerableCode is a free software code scanning tool from nexB Inc. and others.
21+
# Visit https://github.com/nexB/vulnerablecode/ for support and download.
22+
import json
23+
import logging
24+
from builtins import set
25+
from datetime import datetime
26+
from datetime import timezone
27+
from io import BytesIO
28+
from typing import Iterable
29+
from zipfile import ZipFile
30+
31+
import requests
32+
from packageurl import PackageURL
33+
from univers.version_range import PypiVersionRange
34+
from univers.versions import InvalidVersion
35+
from univers.versions import PypiVersion
36+
from univers.versions import SemverVersion
37+
38+
from vulnerabilities.importer import AdvisoryData
39+
from vulnerabilities.importer import AffectedPackage
40+
from vulnerabilities.importer import Importer
41+
from vulnerabilities.importer import Reference
42+
from vulnerabilities.importer import VulnerabilitySeverity
43+
from vulnerabilities.severity_systems import SCORING_SYSTEMS
44+
45+
logger = logging.getLogger(__name__)
46+
47+
48+
class PyPIImporter(Importer):
49+
spdx_license_expression = "CC-BY-4.0"
50+
51+
def advisory_data(self) -> Iterable[AdvisoryData]:
52+
"""
53+
1. Fetch the data from osv api
54+
2. unzip the file
55+
3. open the file one by one
56+
4. yield the json file to parse_advisory_data
57+
"""
58+
url = "https://osv-vulnerabilities.storage.googleapis.com/PyPI/all.zip"
59+
try:
60+
response = requests.get(url).content
61+
with ZipFile(BytesIO(response)) as zip_file:
62+
for file_name in zip_file.namelist():
63+
with zip_file.open(file_name) as f:
64+
vul_info = json.loads(f.read())
65+
yield parse_advisory_data(vul_info)
66+
except requests.exceptions.RequestException:
67+
logger.error("Failed to fetch osv-vulnerabilities PyPI")
68+
69+
70+
def parse_advisory_data(raw_data: dict) -> AdvisoryData:
71+
summary = raw_data.get("summary") or ""
72+
aliases = get_aliases(raw_data)
73+
date_published = get_published_date(raw_data)
74+
severity = get_severity(raw_data)
75+
references = get_references(raw_data, severity)
76+
77+
affected_package = []
78+
if "affected" in raw_data:
79+
for affected_pkg in raw_data["affected"]:
80+
purl = get_aff_purl(affected_pkg, raw_data["id"])
81+
affected_version_range = get_aff_version_range(affected_pkg, raw_data["id"])
82+
83+
for fixed_range in affected_pkg.get("ranges", []):
84+
fixed_version = get_fixed_version(fixed_range, raw_data["id"])
85+
86+
for version in fixed_version:
87+
affected_package.append(
88+
AffectedPackage(
89+
package=purl,
90+
affected_version_range=affected_version_range,
91+
fixed_version=version,
92+
)
93+
)
94+
else:
95+
logger.error(f"affected_package not found - {raw_data['id'] !r}")
96+
97+
return AdvisoryData(
98+
aliases=aliases,
99+
summary=summary,
100+
affected_packages=affected_package,
101+
references=references,
102+
date_published=date_published,
103+
)
104+
105+
106+
def fixed_filter(fixed_range) -> []:
107+
filter_fixed = list(filter(lambda x: x.keys() == {"fixed"}, fixed_range["events"]))
108+
list_fixed = [i["fixed"] for i in filter_fixed]
109+
return list_fixed
110+
111+
112+
def get_aliases(raw_data) -> []:
113+
vulnerability_id = raw_data.get("id")
114+
vulnerability_aliases = raw_data.get("aliases") or []
115+
if vulnerability_id:
116+
vulnerability_aliases.append(vulnerability_id)
117+
return list(dict.fromkeys(vulnerability_aliases))
118+
119+
120+
def get_published_date(raw_data):
121+
if "published" in raw_data:
122+
return datetime.strptime(raw_data["published"][0:19], "%Y-%m-%dT%H:%M:%S").replace(
123+
tzinfo=timezone.utc
124+
)
125+
else:
126+
logger.warning("date_published not found " + raw_data["id"])
127+
128+
129+
def get_severity(raw_data) -> []:
130+
severity = []
131+
if "severity" in raw_data:
132+
for sever_list in raw_data["severity"]:
133+
if "type" in sever_list and sever_list["type"] == "CVSS_V3":
134+
severity.append(
135+
VulnerabilitySeverity(
136+
system=SCORING_SYSTEMS["cvssv3_vector"],
137+
value=sever_list["score"],
138+
)
139+
)
140+
if "ecosystem_specific" in raw_data and "severity" in raw_data["ecosystem_specific"]:
141+
severity.append(
142+
VulnerabilitySeverity(
143+
system=SCORING_SYSTEMS["generic_textual"],
144+
value=raw_data["ecosystem_specific"]["severity"],
145+
)
146+
)
147+
else:
148+
logger.warning(f"severity not found- {raw_data['id']!r}")
149+
150+
return severity
151+
152+
153+
def get_references(raw_data, severity) -> []:
154+
if "references" in raw_data:
155+
return [
156+
Reference(url=ref["url"], severities=severity) for ref in raw_data["references"] if ref
157+
]
158+
else:
159+
return []
160+
161+
162+
def get_aff_purl(affected_pkg, raw_id):
163+
package = affected_pkg["package"]
164+
if "purl" in package:
165+
try:
166+
return PackageURL.from_string(package["purl"])
167+
except ValueError:
168+
logger.error(f"PackageURL ValueError - {raw_id !r} - purl: {package['purl'] !r}")
169+
170+
if "ecosystem" in package and "name" in package:
171+
return PackageURL(type=package["ecosystem"], name=package["name"])
172+
else:
173+
logger.error(f"purl affected_pkg not found - {raw_id !r}")
174+
175+
176+
def get_aff_version_range(affected_pkg, raw_id):
177+
# HINT: Values larger than 1/3 of a buffer page cannot be indexed.
178+
if "versions" in affected_pkg and len(affected_pkg["versions"]) < 200:
179+
try:
180+
return PypiVersionRange(affected_pkg["versions"])
181+
except Exception as e:
182+
logger.error(f"affected_pkg_version_range Error - {raw_id !r} - purl: {e !r}")
183+
184+
185+
def get_fixed_version(fixed_range, raw_id) -> {}:
186+
fixed_version = set()
187+
if "type" in fixed_range:
188+
list_fixed = fixed_filter(fixed_range)
189+
for i in list_fixed:
190+
if fixed_range["type"] == "ECOSYSTEM":
191+
try:
192+
fixed_version.add(PypiVersion(i))
193+
except InvalidVersion:
194+
logger.error(f"Invalid Version - PypiVersion - {raw_id !r} - {i !r}")
195+
if fixed_range["type"] == "SEMVER":
196+
try:
197+
fixed_version.add(SemverVersion(i))
198+
except InvalidVersion:
199+
logger.error(f"Invalid Version - SemverVersion - {raw_id !r} - {i !r}")
200+
if fixed_range["type"] == "GIT":
201+
# TODO add GitHubVersion univers fix_version
202+
logger.error(f"NotImplementedError GIT Version - {raw_id !r} - {i !r}")
203+
else:
204+
logger.error(f"Invalid type - {raw_id!r}")
205+
206+
return fixed_version

0 commit comments

Comments
 (0)