Skip to content

Commit 550a765

Browse files
ziadhanyziadhany
committed
Resolve merge conflict
Use the public facing URL Simplify fetch git repo Remove the / from the end of cvss3 vector Try to make github-review filter less complex Add a supported_ecosystems as a list Remove try catch for version_class Use the appropriate exception CVSS3MalformedError Add GithubOSVImprover to IMPROVERS_REGISTRY Resolve merge conflict Update univers version Fix github_osv expected values Add a ruby version test Fix merge conflict Add support for nuget , hex , gem , cargo Add support for maven , go and packagist ecosystem. Import data from github advisory-database using osv format and add support for all osv ecosystems Signed-off-by: ziadhany <ziadhany2016@gmail.com>
1 parent 37fdd7d commit 550a765

18 files changed

Lines changed: 1114 additions & 27 deletions

‎vulnerabilities/importers/__init__.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
from vulnerabilities.importers import fireeye
1919
from vulnerabilities.importers import gentoo
2020
from vulnerabilities.importers import github
21+
from vulnerabilities.importers import github_osv
2122
from vulnerabilities.importers import gitlab
2223
from vulnerabilities.importers import istio
2324
from vulnerabilities.importers import mozilla
@@ -65,6 +66,7 @@
6566
ubuntu_usn.UbuntuUSNImporter,
6667
fireeye.FireyeImporter,
6768
apache_kafka.ApacheKafkaImporter,
69+
github_osv.GithubOSVImporter,
6870
]
6971

7072
IMPORTERS_REGISTRY = {x.qualified_name: x for x in IMPORTERS_REGISTRY}

‎vulnerabilities/importers/github_osv.py‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
#
2+
# Copyright (c) nexB Inc. and others. All rights reserved.
3+
# VulnerableCode is a trademark of nexB Inc.
4+
# SPDX-License-Identifier: Apache-2.0
5+
# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
6+
# See https://github.com/nexB/vulnerablecode for support or download.
7+
# See https://aboutcode.org for more information about nexB OSS projects.
8+
#
9+
import json
10+
import logging
11+
from io import BytesIO
12+
from pathlib import Path
13+
from typing import Iterable
14+
from zipfile import ZipFile
15+
16+
import requests
17+
18+
from vulnerabilities.importer import AdvisoryData
19+
from vulnerabilities.importer import Importer
20+
from vulnerabilities.importers.osv import parse_advisory_data
21+
22+
logger = logging.getLogger(__name__)
23+
24+
25+
class GithubOSVImporter(Importer):
26+
license_url = "https://github.com/github/advisory-database/blob/main/LICENSE.md"
27+
spdx_license_expression = "CC-BY-4.0"
28+
repo_url = "git+https://github.com/github/advisory-database/"
29+
30+
def advisory_data(self) -> Iterable[AdvisoryData]:
31+
supported_ecosystems = [
32+
"pypi",
33+
"npm",
34+
"maven",
35+
"golang",
36+
"composer",
37+
"hex",
38+
"gem",
39+
"nuget",
40+
"cargo",
41+
]
42+
try:
43+
self.clone(repo_url=self.repo_url)
44+
path = Path(self.vcs_response.dest_dir)
45+
# filter out non-github-reviewed files and only keep the files end-with .json
46+
advisory_dirs = path / "advisories/github-reviewed"
47+
for file in advisory_dirs.glob("**/*.json"):
48+
with open(file) as f:
49+
raw_data = json.load(f)
50+
yield parse_advisory_data(raw_data, supported_ecosystems)
51+
finally:
52+
if self.vcs_response:
53+
self.vcs_response.delete()

‎vulnerabilities/importers/osv.py‎

Lines changed: 53 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -13,10 +13,10 @@
1313
from typing import Optional
1414

1515
import dateparser
16+
from cvss.exceptions import CVSS3MalformedError
1617
from packageurl import PackageURL
1718
from univers.version_range import RANGE_CLASS_BY_SCHEMES
1819
from univers.versions import InvalidVersion
19-
from univers.versions import PypiVersion
2020
from univers.versions import SemverVersion
2121
from univers.versions import Version
2222

@@ -30,8 +30,20 @@
3030

3131
logger = logging.getLogger(__name__)
3232

33+
PURL_TYPE_BY_OSV_SCHEME = {
34+
"npm": "npm",
35+
"pypi": "pypi",
36+
"maven": "maven",
37+
"nuget": "nuget",
38+
"packagist": "composer",
39+
"rubygems": "gem",
40+
"go": "golang",
41+
"hex": "hex",
42+
"cargo": "cargo",
43+
}
3344

34-
def parse_advisory_data(raw_data: dict, supported_ecosystem) -> Optional[AdvisoryData]:
45+
46+
def parse_advisory_data(raw_data: dict, supported_ecosystems: List) -> Optional[AdvisoryData]:
3547
"""
3648
Return an AdvisoryData build from a ``raw_data`` mapping of OSV advisory and
3749
a ``supported_ecosystem`` string.
@@ -53,18 +65,24 @@ def parse_advisory_data(raw_data: dict, supported_ecosystem) -> Optional[Advisor
5365

5466
for affected_pkg in raw_data.get("affected") or []:
5567
purl = get_affected_purl(affected_pkg=affected_pkg, raw_id=raw_id)
56-
if purl.type != supported_ecosystem:
68+
if purl.type in PURL_TYPE_BY_OSV_SCHEME:
69+
new_type = PURL_TYPE_BY_OSV_SCHEME[purl.type]
70+
purl = purl._replace(type=new_type)
71+
72+
if purl.type not in supported_ecosystems:
5773
logger.error(f"Unsupported package type: {purl!r} in OSV: {raw_id!r}")
5874
continue
5975

6076
affected_version_range = get_affected_version_range(
6177
affected_pkg=affected_pkg,
6278
raw_id=raw_id,
63-
supported_ecosystem=supported_ecosystem,
79+
supported_ecosystem=purl.type,
6480
)
6581

6682
for fixed_range in affected_pkg.get("ranges") or []:
67-
fixed_version = get_fixed_versions(fixed_range=fixed_range, raw_id=raw_id)
83+
fixed_version = get_fixed_versions(
84+
fixed_range=fixed_range, raw_id=raw_id, supported_ecosystem=purl.type
85+
)
6886

6987
for version in fixed_version:
7088
affected_packages.append(
@@ -74,7 +92,6 @@ def parse_advisory_data(raw_data: dict, supported_ecosystem) -> Optional[Advisor
7492
fixed_version=version,
7593
)
7694
)
77-
7895
return AdvisoryData(
7996
aliases=aliases,
8097
summary=summary,
@@ -113,14 +130,21 @@ def get_severities(raw_data) -> Iterable[VulnerabilitySeverity]:
113130
"""
114131
Yield VulnerabilitySeverity extracted from a mapping of OSV ``raw_data``
115132
"""
116-
for severity in raw_data.get("severity") or []:
117-
if severity.get("type") == "CVSS_V3":
118-
vector = severity["score"]
119-
system = SCORING_SYSTEMS["cvssv3.1"]
120-
score = system.compute(vector)
121-
yield VulnerabilitySeverity(system=system, value=score, scoring_elements=vector)
122-
else:
123-
logger.error(f"Unsupported severity type: {severity!r} for OSV id: {raw_data['id']!r}")
133+
try:
134+
for severity in raw_data.get("severity") or []:
135+
if severity.get("type") == "CVSS_V3":
136+
vector = severity["score"]
137+
valid_vector = vector[::-1] if vector[-1] == "/" else vector
138+
system = SCORING_SYSTEMS["cvssv3.1"]
139+
score = system.compute(valid_vector)
140+
yield VulnerabilitySeverity(system=system, value=score, scoring_elements=vector)
141+
142+
else:
143+
logger.error(
144+
f"Unsupported severity type: {severity!r} for OSV id: {raw_data['id']!r}"
145+
)
146+
except CVSS3MalformedError as e:
147+
logger.error(f"Invalid severity {e}")
124148

125149
ecosystem_specific = raw_data.get("ecosystem_specific") or {}
126150
severity = ecosystem_specific.get("severity")
@@ -199,18 +223,19 @@ def get_affected_version_range(affected_pkg, raw_id, supported_ecosystem):
199223
)
200224

201225

202-
def get_fixed_versions(fixed_range, raw_id) -> List[Version]:
226+
def get_fixed_versions(fixed_range, raw_id, supported_ecosystem) -> List[Version]:
203227
"""
204228
Return a list of unique fixed univers Versions given a ``fixed_range``
205229
univers VersionRange and a ``raw_id``.
206230
207231
For example::
208232
209-
>>> get_fixed_versions(fixed_range={}, raw_id="GHSA-j3f7-7rmc-6wqj")
233+
>>> get_fixed_versions(fixed_range={}, raw_id="GHSA-j3f7-7rmc-6wqj", supported_ecosystem="pypi",)
210234
[]
211235
>>> get_fixed_versions(
212236
... fixed_range={"type": "ECOSYSTEM", "events": [{"fixed": "1.7.0"}]},
213-
... raw_id="GHSA-j3f7-7rmc-6wqj"
237+
... raw_id="GHSA-j3f7-7rmc-6wqj",
238+
... supported_ecosystem="pypi",
214239
... )
215240
[PypiVersion(string='1.7.0')]
216241
"""
@@ -221,21 +246,27 @@ def get_fixed_versions(fixed_range, raw_id) -> List[Version]:
221246

222247
fixed_range_type = fixed_range["type"]
223248

224-
for version in extract_fixed_versions(fixed_range):
249+
version_range_class = RANGE_CLASS_BY_SCHEMES.get(supported_ecosystem)
250+
version_class = version_range_class.version_class if version_range_class else None
225251

226-
# FIXME: ECOSYSTEM does not imply PyPI!!!!
252+
for version in extract_fixed_versions(fixed_range):
227253
if fixed_range_type == "ECOSYSTEM":
228254
try:
229-
fixed_versions.append(PypiVersion(version))
255+
if not version_class:
256+
raise InvalidVersion(
257+
f"Unsupported version for ecosystem: {supported_ecosystem}"
258+
)
259+
fixed_versions.append(version_class(version))
230260
except InvalidVersion:
231-
logger.error(f"Invalid PypiVersion: {version!r} for OSV id: {raw_id!r}")
261+
logger.error(
262+
f"Invalid version class: {version_class} - {version!r} for OSV id: {raw_id!r}"
263+
)
232264

233265
elif fixed_range_type == "SEMVER":
234266
try:
235267
fixed_versions.append(SemverVersion(version))
236268
except InvalidVersion:
237269
logger.error(f"Invalid SemverVersion: {version!r} for OSV id: {raw_id!r}")
238-
239270
else:
240271
logger.error(f"Unsupported fixed version type: {version!r} for OSV id: {raw_id!r}")
241272

‎vulnerabilities/importers/pypa.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ def advisory_data(self) -> Iterable[AdvisoryData]:
3131
self.clone(repo_url=self.repo_url)
3232
path = Path(self.vcs_response.dest_dir)
3333
for raw_data in fork_and_get_files(path=path):
34-
yield parse_advisory_data(raw_data=raw_data, supported_ecosystem="pypi")
34+
yield parse_advisory_data(raw_data=raw_data, supported_ecosystems=["pypi"])
3535
finally:
3636
if self.vcs_response:
3737
self.vcs_response.delete()

‎vulnerabilities/importers/pysec.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,4 +38,4 @@ def advisory_data(self) -> Iterable[AdvisoryData]:
3838
continue
3939
with zip_file.open(file_name) as f:
4040
vul_info = json.load(f)
41-
yield parse_advisory_data(raw_data=vul_info, supported_ecosystem="pypi")
41+
yield parse_advisory_data(raw_data=vul_info, supported_ecosystems=["pypi"])

‎vulnerabilities/improvers/__init__.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
valid_versions.IstioImprover,
2323
valid_versions.DebianOvalImprover,
2424
valid_versions.UbuntuOvalImprover,
25+
valid_versions.GithubOSVImprover,
2526
]
2627

2728
IMPROVERS_REGISTRY = {x.qualified_name: x for x in IMPROVERS_REGISTRY}

‎vulnerabilities/improvers/valid_versions.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131
from vulnerabilities.importers.debian_oval import DebianOvalImporter
3232
from vulnerabilities.importers.elixir_security import ElixirSecurityImporter
3333
from vulnerabilities.importers.github import GitHubAPIImporter
34+
from vulnerabilities.importers.github_osv import GithubOSVImporter
3435
from vulnerabilities.importers.gitlab import GitLabAPIImporter
3536
from vulnerabilities.importers.istio import IstioImporter
3637
from vulnerabilities.importers.nginx import NginxImporter
@@ -477,3 +478,8 @@ class DebianOvalImprover(ValidVersionImprover):
477478
class UbuntuOvalImprover(ValidVersionImprover):
478479
importer = UbuntuImporter
479480
ignorable_versions = []
481+
482+
483+
class GithubOSVImprover(ValidVersionImprover):
484+
importer = GithubOSVImporter
485+
ignorable_versions = []

‎vulnerabilities/tests/test_data/github_osv/github_osv_expected_1.json‎

Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,121 @@
1+
{
2+
"aliases": [
3+
"CVE-2015-8315",
4+
"GHSA-3fx5-fwvr-xrjg"
5+
],
6+
"summary": "Regular Expression Denial of Service in ms\nVersions of `ms` prior to 0.7.1 are affected by a regular expression denial of service vulnerability when extremely long version strings are parsed.\n\n## Proof of Concept\n```javascript\nvar ms = require('ms');\nvar genstr = function (len, chr) {\n var result = \"\";\n for (i=0; i<=len; i++) {\n result = result + chr;\n }\n\n return result;\n}\n\nms(genstr(process.argv[2], \"5\") + \" minutea\");\n\n```\n\n### Results\nShowing increase in execution time based on the input string.\n```\n$ time node ms.js 10000\n\nreal\t0m0.758s\nuser\t0m0.724s\nsys\t0m0.031s\n\n$ time node ms.js 20000\n\nreal\t0m2.580s\nuser\t0m2.494s\nsys\t0m0.047s\n\n$ time node ms.js 30000\n\nreal\t0m5.747s\nuser\t0m5.483s\nsys\t0m0.080s\n\n$ time node ms.js 80000\n\nreal\t0m41.022s\nuser\t0m38.894s\nsys\t0m0.529s\n```",
7+
"affected_packages": [
8+
{
9+
"package": {
10+
"type": "npm",
11+
"namespace": null,
12+
"name": "ms",
13+
"version": null,
14+
"qualifiers": null,
15+
"subpath": null
16+
},
17+
"affected_version_range": null,
18+
"fixed_version": "0.7.1"
19+
}
20+
],
21+
"references": [
22+
{
23+
"reference_id": "",
24+
"url": "https://nvd.nist.gov/vuln/detail/CVE-2015-8315",
25+
"severities": [
26+
{
27+
"system": "cvssv3.1",
28+
"value": "7.5",
29+
"scoring_elements": "CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H"
30+
},
31+
{
32+
"system": "generic_textual",
33+
"value": "HIGH",
34+
"scoring_elements": ""
35+
}
36+
]
37+
},
38+
{
39+
"reference_id": "",
40+
"url": "https://github.com/unshiftio/millisecond/",
41+
"severities": [
42+
{
43+
"system": "cvssv3.1",
44+
"value": "7.5",
45+
"scoring_elements": "CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H"
46+
},
47+
{
48+
"system": "generic_textual",
49+
"value": "HIGH",
50+
"scoring_elements": ""
51+
}
52+
]
53+
},
54+
{
55+
"reference_id": "",
56+
"url": "https://support.f5.com/csp/article/K46337613?utm_source=f5support&amp;utm_medium=RSS",
57+
"severities": [
58+
{
59+
"system": "cvssv3.1",
60+
"value": "7.5",
61+
"scoring_elements": "CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H"
62+
},
63+
{
64+
"system": "generic_textual",
65+
"value": "HIGH",
66+
"scoring_elements": ""
67+
}
68+
]
69+
},
70+
{
71+
"reference_id": "",
72+
"url": "https://www.npmjs.com/advisories/46",
73+
"severities": [
74+
{
75+
"system": "cvssv3.1",
76+
"value": "7.5",
77+
"scoring_elements": "CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H"
78+
},
79+
{
80+
"system": "generic_textual",
81+
"value": "HIGH",
82+
"scoring_elements": ""
83+
}
84+
]
85+
},
86+
{
87+
"reference_id": "",
88+
"url": "http://www.openwall.com/lists/oss-security/2016/04/20/11",
89+
"severities": [
90+
{
91+
"system": "cvssv3.1",
92+
"value": "7.5",
93+
"scoring_elements": "CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H"
94+
},
95+
{
96+
"system": "generic_textual",
97+
"value": "HIGH",
98+
"scoring_elements": ""
99+
}
100+
]
101+
},
102+
{
103+
"reference_id": "",
104+
"url": "http://www.securityfocus.com/bid/96389",
105+
"severities": [
106+
{
107+
"system": "cvssv3.1",
108+
"value": "7.5",
109+
"scoring_elements": "CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H"
110+
},
111+
{
112+
"system": "generic_textual",
113+
"value": "HIGH",
114+
"scoring_elements": ""
115+
}
116+
]
117+
}
118+
],
119+
"date_published": "2017-10-24T18:33:36+00:00",
120+
"weaknesses": []
121+
}

0 commit comments

Comments
 (0)