Skip to content

Commit ab10ca0

Browse files
pombredannepombredanne
authored
Merge pull request #789 from keshav-space/deps_validator
add DepsDataSource
2 parents 07a7434 + 5cdf24a commit ab10ca0

10 files changed

Lines changed: 1057 additions & 0 deletions

‎vulntotal/datasources/deps.py‎

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
#
2+
# Copyright (c) nexB Inc. and others. All rights reserved.
3+
# VulnerableCode is a trademark of nexB Inc.
4+
# SPDX-License-Identifier: Apache-2.0
5+
# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
6+
# See https://github.com/nexB/vulnerablecode for support or download.
7+
# See https://aboutcode.org for more information about nexB OSS projects.
8+
#
9+
10+
import json
11+
import logging
12+
from typing import Iterable
13+
from urllib.parse import quote
14+
15+
import requests
16+
from packageurl import PackageURL
17+
18+
from vulntotal.validator import DataSource
19+
from vulntotal.validator import VendorData
20+
21+
logger = logging.getLogger(__name__)
22+
23+
24+
class DepsDataSource(DataSource):
25+
spdx_license_expression = "TODO"
26+
license_url = "TODO"
27+
28+
def fetch_json_response(self, url):
29+
response = requests.get(url)
30+
if not response.status_code == 200 or response.text == "Not Found":
31+
logger.error(f"Error while fetching {url}")
32+
return
33+
return response.json()
34+
35+
def datasource_advisory(self, purl) -> Iterable[VendorData]:
36+
payload = generate_meta_payload(purl)
37+
response = self.fetch_json_response(payload)
38+
if response:
39+
advisories = parse_advisories_from_meta(response)
40+
if advisories:
41+
for advisory in advisories:
42+
advisory_payload = generate_advisory_payload(advisory)
43+
fetched_advisory = self.fetch_json_response(advisory_payload)
44+
self._raw_dump.append(fetched_advisory)
45+
if fetched_advisory:
46+
return parse_advisory(fetched_advisory)
47+
48+
@classmethod
49+
def supported_ecosystem(cls):
50+
return {
51+
"npm": "npm",
52+
"maven": "maven",
53+
"golang": "go",
54+
"pypi": "pypi",
55+
"cargo": "cargo",
56+
# Coming soon
57+
# "nuget": "nuget",
58+
}
59+
60+
61+
def parse_advisory(advisory) -> Iterable[VendorData]:
62+
affected_versions = [event["version"] for event in advisory["packages"][0]["versionsAffected"]]
63+
fixed_versions = [event["version"] for event in advisory["packages"][0]["versionsUnaffected"]]
64+
yield VendorData(
65+
aliases=sorted(list(set(advisory["aliases"]))),
66+
affected_versions=sorted(list(set(affected_versions))),
67+
fixed_versions=sorted(list(set(fixed_versions))),
68+
)
69+
70+
71+
def parse_advisories_from_meta(advisories_metadata):
72+
advisories = []
73+
if "dependencies" in advisories_metadata and advisories_metadata["dependencies"]:
74+
for dependency in advisories_metadata["dependencies"]:
75+
if dependency["advisories"]:
76+
advisories.extend(dependency["advisories"])
77+
return advisories
78+
79+
80+
def generate_advisory_payload(advisory_meta):
81+
url_advisory = "https://deps.dev/_/advisory/{source}/{sourceID}"
82+
return url_advisory.format(source=advisory_meta["source"], sourceID=advisory_meta["sourceID"])
83+
84+
85+
def generate_meta_payload(purl):
86+
url_advisories_meta = "https://deps.dev/_/s/{ecosystem}/p/{package}/v/{version}/dependencies"
87+
supported_ecosystem = DepsDataSource.supported_ecosystem()
88+
if purl.type in supported_ecosystem:
89+
purl_version = purl.version
90+
purl_name = purl.name
91+
92+
if purl.type == "maven":
93+
if not purl.namespace:
94+
logger.error(f"Invalid Maven PURL {str(purl)}")
95+
return
96+
purl_name = quote(f"{purl.namespace}:{purl.name}", safe="")
97+
98+
elif purl.type == "golang":
99+
if purl.namespace:
100+
purl_name = quote(f"{purl.namespace}/{purl.name}", safe="")
101+
if not purl_version.startswith("v"):
102+
purl_version = f"v{purl_version}"
103+
104+
return url_advisories_meta.format(
105+
ecosystem=supported_ecosystem[purl.type],
106+
package=purl_name,
107+
version=purl_version,
108+
)

‎vulntotal/tests/test_data/deps/advisories_metadata.json‎

Lines changed: 179 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,179 @@
1+
[
2+
{
3+
"source": "GHSA",
4+
"sourceID": "GHSA-g3rq-g295-4j3m",
5+
"sourceURL": "https://github.com/advisories/GHSA-g3rq-g295-4j3m",
6+
"title": "Regular Expression Denial of Service (ReDoS) in Jinja2",
7+
"description": "This affects the package jinja2 from 0.0.0 and before 2.11.3. The ReDOS vulnerability of the regex is mainly due to the sub-pattern [a-zA-Z0-9._-]+.[a-zA-Z0-9._-]+ This issue can be mitigated by Markdown to format user content instead of the urlize filter, or by implementing request timeouts and limiting process memory.",
8+
"referenceURLs": [
9+
"https://nvd.nist.gov/vuln/detail/CVE-2020-28493",
10+
"https://github.com/pallets/jinja/pull/1343",
11+
"https://github.com/pallets/jinja/blob/ab81fd9c277900c85da0c322a2ff9d68a235b2e6/src/jinja2/utils.py%23L20",
12+
"https://snyk.io/vuln/SNYK-PYTHON-JINJA2-1012994",
13+
"https://lists.fedoraproject.org/archives/list/package-announce@lists.fedoraproject.org/message/PVAKCOO7VBVUBM3Q6CBBTPBFNP5NDXF4/",
14+
"https://security.gentoo.org/glsa/202107-19",
15+
"https://github.com/advisories/GHSA-g3rq-g295-4j3m"
16+
],
17+
"severity": "MEDIUM",
18+
"gitHubSeverity": "MODERATE",
19+
"scoreV3": 5.3,
20+
"aliases": [
21+
"CVE-2020-28493"
22+
],
23+
"disclosedAt": 1616189285,
24+
"observedAt": 1650328213
25+
},
26+
{
27+
"source": "OSV",
28+
"sourceID": "PYSEC-2014-8",
29+
"sourceURL": "https://osv.dev/vulnerability/PYSEC-2014-8",
30+
"title": "PYSEC-2014-8",
31+
"description": "The default configuration for bccache.FileSystemBytecodeCache in Jinja2 before 2.7.2 does not properly create temporary files, which allows local users to gain privileges via a crafted .cache file with a name starting with __jinja2_ in /tmp.",
32+
"referenceURLs": [
33+
"http://advisories.mageia.org/MGASA-2014-0028.html",
34+
"http://jinja.pocoo.org/docs/changelog/",
35+
"http://openwall.com/lists/oss-security/2014/01/10/2",
36+
"http://openwall.com/lists/oss-security/2014/01/10/3",
37+
"http://rhn.redhat.com/errata/RHSA-2014-0747.html",
38+
"http://rhn.redhat.com/errata/RHSA-2014-0748.html",
39+
"http://secunia.com/advisories/56287",
40+
"http://secunia.com/advisories/58783",
41+
"http://secunia.com/advisories/58918",
42+
"http://secunia.com/advisories/59017",
43+
"http://secunia.com/advisories/60738",
44+
"http://secunia.com/advisories/60770",
45+
"http://www.gentoo.org/security/en/glsa/glsa-201408-13.xml",
46+
"http://www.mandriva.com/security/advisories?name=MDVSA-2014:096",
47+
"https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=734747",
48+
"https://bugzilla.redhat.com/show_bug.cgi?id=1051421",
49+
"https://github.com/pypa/advisory-database/blob/main/vulns/jinja2/PYSEC-2014-8.yaml",
50+
"https://oss.oracle.com/pipermail/el-errata/2014-June/004192.html"
51+
],
52+
"severity": "UNKNOWN",
53+
"gitHubSeverity": "UNKNOWN",
54+
"aliases": [
55+
"CVE-2014-1402"
56+
],
57+
"disclosedAt": 1400511300,
58+
"observedAt": 1645597812
59+
},
60+
{
61+
"source": "OSV",
62+
"sourceID": "PYSEC-2014-82",
63+
"sourceURL": "https://osv.dev/vulnerability/PYSEC-2014-82",
64+
"title": "PYSEC-2014-82",
65+
"description": "FileSystemBytecodeCache in Jinja2 2.7.2 does not properly create temporary directories, which allows local users to gain privileges by pre-creating a temporary directory with a user's uid. NOTE: this vulnerability exists because of an incomplete fix for CVE-2014-1402.",
66+
"referenceURLs": [
67+
"http://seclists.org/oss-sec/2014/q1/73",
68+
"http://secunia.com/advisories/56328",
69+
"http://secunia.com/advisories/60738",
70+
"http://www.gentoo.org/security/en/glsa/glsa-201408-13.xml",
71+
"https://bugzilla.redhat.com/show_bug.cgi?id=1051421",
72+
"https://github.com/mitsuhiko/jinja2",
73+
"https://github.com/mitsuhiko/jinja2/commit/acb672b6a179567632e032f547582f30fa2f4aa7",
74+
"https://github.com/mitsuhiko/jinja2/pull/292",
75+
"https://github.com/mitsuhiko/jinja2/pull/296",
76+
"https://github.com/pypa/advisory-database/blob/main/vulns/jinja2/PYSEC-2014-82.yaml"
77+
],
78+
"severity": "UNKNOWN",
79+
"gitHubSeverity": "UNKNOWN",
80+
"aliases": [
81+
"CVE-2014-0012"
82+
],
83+
"disclosedAt": 1400511300,
84+
"observedAt": 1645597812
85+
},
86+
{
87+
"source": "OSV",
88+
"sourceID": "PYSEC-2019-217",
89+
"sourceURL": "https://osv.dev/vulnerability/PYSEC-2019-217",
90+
"title": "PYSEC-2019-217",
91+
"description": "In Pallets Jinja before 2.10.1, str.format_map allows a sandbox escape.",
92+
"referenceURLs": [
93+
"http://lists.opensuse.org/opensuse-security-announce/2019-05/msg00030.html",
94+
"http://lists.opensuse.org/opensuse-security-announce/2019-06/msg00064.html",
95+
"https://access.redhat.com/errata/RHSA-2019:1152",
96+
"https://access.redhat.com/errata/RHSA-2019:1237",
97+
"https://access.redhat.com/errata/RHSA-2019:1329",
98+
"https://github.com/advisories/GHSA-462w-v97r-4m45",
99+
"https://github.com/pypa/advisory-database/blob/main/vulns/jinja2/PYSEC-2019-217.yaml",
100+
"https://lists.apache.org/thread.html/09fc842ff444cd43d9d4c510756fec625ef8eb1175f14fd21de2605f@%3Cdevnull.infra.apache.org%3E",
101+
"https://lists.apache.org/thread.html/2b52b9c8b9d6366a4f1b407a8bde6af28d9fc73fdb3b37695fd0d9ac@%3Cdevnull.infra.apache.org%3E",
102+
"https://lists.apache.org/thread.html/320441dccbd9a545320f5f07306d711d4bbd31ba43dc9eebcfc602df@%3Cdevnull.infra.apache.org%3E",
103+
"https://lists.apache.org/thread.html/46c055e173b52d599c648a98199972dbd6a89d2b4c4647b0500f2284@%3Cdevnull.infra.apache.org%3E",
104+
"https://lists.apache.org/thread.html/57673a78c4d5c870d3f21465c7e2946b9f8285c7c57e54c2ae552f02@%3Ccommits.airflow.apache.org%3E",
105+
"https://lists.apache.org/thread.html/7f39f01392d320dfb48e4901db68daeece62fd60ef20955966739993@%3Ccommits.airflow.apache.org%3E",
106+
"https://lists.apache.org/thread.html/b2380d147b508bbcb90d2cad443c159e63e12555966ab4f320ee22da@%3Ccommits.airflow.apache.org%3E",
107+
"https://lists.apache.org/thread.html/f0c4a03418bcfe70c539c5dbaf99c04c98da13bfa1d3266f08564316@%3Ccommits.airflow.apache.org%3E",
108+
"https://lists.fedoraproject.org/archives/list/package-announce@lists.fedoraproject.org/message/DSW3QZMFVVR7YE3UT4YRQA272TYAL5AF/",
109+
"https://lists.fedoraproject.org/archives/list/package-announce@lists.fedoraproject.org/message/QCDYIS254EJMBNWOG4S5QY6AOTOR4TZU/",
110+
"https://lists.fedoraproject.org/archives/list/package-announce@lists.fedoraproject.org/message/TS7IVZAJBWOHNRDMFJDIZVFCMRP6YIUQ/",
111+
"https://palletsprojects.com/blog/jinja-2-10-1-released",
112+
"https://usn.ubuntu.com/4011-1/",
113+
"https://usn.ubuntu.com/4011-2/"
114+
],
115+
"severity": "UNKNOWN",
116+
"gitHubSeverity": "UNKNOWN",
117+
"aliases": [
118+
"CVE-2019-10906",
119+
"GHSA-462w-v97r-4m45"
120+
],
121+
"disclosedAt": 1554596940,
122+
"observedAt": 1645597812
123+
},
124+
{
125+
"source": "OSV",
126+
"sourceID": "PYSEC-2019-220",
127+
"sourceURL": "https://osv.dev/vulnerability/PYSEC-2019-220",
128+
"title": "PYSEC-2019-220",
129+
"description": "In Pallets Jinja before 2.8.1, str.format allows a sandbox escape.",
130+
"referenceURLs": [
131+
"http://lists.opensuse.org/opensuse-security-announce/2019-05/msg00030.html",
132+
"http://lists.opensuse.org/opensuse-security-announce/2019-06/msg00064.html",
133+
"https://access.redhat.com/errata/RHSA-2019:1022",
134+
"https://access.redhat.com/errata/RHSA-2019:1237",
135+
"https://access.redhat.com/errata/RHSA-2019:1260",
136+
"https://access.redhat.com/errata/RHSA-2019:3964",
137+
"https://access.redhat.com/errata/RHSA-2019:4062",
138+
"https://github.com/advisories/GHSA-hj2j-77xm-mc5v",
139+
"https://github.com/pallets/jinja",
140+
"https://github.com/pallets/jinja/commit/9b53045c34e61013dc8f09b7e52a555fa16bed16",
141+
"https://github.com/pypa/advisory-database/blob/main/vulns/jinja2/PYSEC-2019-220.yaml",
142+
"https://palletsprojects.com/blog/jinja-281-released/",
143+
"https://usn.ubuntu.com/4011-1/",
144+
"https://usn.ubuntu.com/4011-2/"
145+
],
146+
"severity": "UNKNOWN",
147+
"gitHubSeverity": "UNKNOWN",
148+
"aliases": [
149+
"CVE-2016-10745",
150+
"GHSA-hj2j-77xm-mc5v"
151+
],
152+
"disclosedAt": 1554730140,
153+
"observedAt": 1645597812
154+
},
155+
{
156+
"source": "OSV",
157+
"sourceID": "PYSEC-2021-66",
158+
"sourceURL": "https://osv.dev/vulnerability/PYSEC-2021-66",
159+
"title": "PYSEC-2021-66",
160+
"description": "This affects the package jinja2 from 0.0.0 and before 2.11.3. The ReDoS vulnerability is mainly due to the `_punctuation_re regex` operator and its use of multiple wildcards. The last wildcard is the most exploitable as it searches for trailing punctuation. This issue can be mitigated by Markdown to format user content instead of the urlize filter, or by implementing request timeouts and limiting process memory.",
161+
"referenceURLs": [
162+
"https://github.com/advisories/GHSA-g3rq-g295-4j3m",
163+
"https://github.com/pallets/jinja/blob/ab81fd9c277900c85da0c322a2ff9d68a235b2e6/src/jinja2/utils.py%23L20",
164+
"https://github.com/pallets/jinja/pull/1343",
165+
"https://github.com/pypa/advisory-database/blob/main/vulns/jinja2/PYSEC-2021-66.yaml",
166+
"https://lists.fedoraproject.org/archives/list/package-announce@lists.fedoraproject.org/message/PVAKCOO7VBVUBM3Q6CBBTPBFNP5NDXF4/",
167+
"https://snyk.io/vuln/SNYK-PYTHON-JINJA2-1012994"
168+
],
169+
"severity": "UNKNOWN",
170+
"gitHubSeverity": "UNKNOWN",
171+
"aliases": [
172+
"CVE-2020-28493",
173+
"SNYK-PYTHON-JINJA2-1012994",
174+
"GHSA-g3rq-g295-4j3m"
175+
],
176+
"disclosedAt": 1612210500,
177+
"observedAt": 1645597812
178+
}
179+
]

0 commit comments

Comments
 (0)