Skip to content

Commit 5e55167

Browse files
sbs2001sbs2001
authored
Merge pull request #292 from sbs2001/import_apache_tomcat
Import apache tomcat
2 parents 322bf12 + 2795792 commit 5e55167

5 files changed

Lines changed: 527 additions & 0 deletions

File tree

‎vulnerabilities/importer_yielder.py‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -226,6 +226,15 @@
226226
'data_source': 'PostgreSQLDataSource',
227227
'data_source_cfg': {},
228228
},
229+
{
230+
'name': 'apache_tomcat',
231+
'license': '',
232+
'last_run': None,
233+
'data_source': 'ApacheTomcatDataSource',
234+
'data_source_cfg': {
235+
"etags": {}
236+
},
237+
},
229238

230239
]
231240

‎vulnerabilities/importers/__init__.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,3 +43,4 @@
4343
from vulnerabilities.importers.kaybee import KaybeeDataSource
4444
from vulnerabilities.importers.nginx import NginxDataSource
4545
from vulnerabilities.importers.postgresql import PostgreSQLDataSource
46+
from vulnerabilities.importers.apache_tomcat import ApacheTomcatDataSource

‎vulnerabilities/importers/apache_tomcat.py‎

Lines changed: 150 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,150 @@
1+
# Copyright (c) nexB Inc. and others. All rights reserved.
2+
# http://nexb.com and https://github.com/nexB/vulnerablecode/
3+
# The VulnerableCode software is licensed under the Apache License version 2.0.
4+
# Data generated with VulnerableCode require an acknowledgment.
5+
#
6+
# You may not use this software except in compliance with the License.
7+
# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
8+
# Unless required by applicable law or agreed to in writing, software distributed
9+
# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
10+
# CONDITIONS OF ANY KIND, either express or implied. See the License for the
11+
# specific language governing permissions and limitations under the License.
12+
#
13+
# When you publish or redistribute any data created with VulnerableCode or any VulnerableCode
14+
# derivative work, you must accompany this data with the following acknowledgment:
15+
#
16+
# Generated with VulnerableCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES
17+
# OR CONDITIONS OF ANY KIND, either express or implied. No content created from
18+
# VulnerableCode should be considered or used as legal advice. Consult an Attorney
19+
# for any legal advice.
20+
# VulnerableCode is a free software tool from nexB Inc. and others.
21+
# Visit https://github.com/nexB/vulnerablecode/ for support and download.
22+
23+
import asyncio
24+
import dataclasses
25+
import re
26+
27+
28+
import requests
29+
from bs4 import BeautifulSoup
30+
from dephell_specifier import RangeSpecifier
31+
from packageurl import PackageURL
32+
33+
from vulnerabilities.data_source import Advisory
34+
from vulnerabilities.data_source import DataSource
35+
from vulnerabilities.data_source import DataSourceConfiguration
36+
from vulnerabilities.data_source import Reference
37+
from vulnerabilities.helpers import create_etag
38+
from vulnerabilities.package_managers import MavenVersionAPI
39+
40+
41+
@dataclasses.dataclass
42+
class ApacheTomcatDataSourceConfiguration(DataSourceConfiguration):
43+
etags: dict
44+
45+
46+
class ApacheTomcatDataSource(DataSource):
47+
48+
CONFIG_CLASS = ApacheTomcatDataSourceConfiguration
49+
base_url = "https://tomcat.apache.org/security-{}"
50+
51+
def __init__(self, *args, **kwargs):
52+
super().__init__(*args, **kwargs)
53+
self.version_api = MavenVersionAPI()
54+
asyncio.run(self.version_api.load_api({"org.apache.tomcat:tomcat"}))
55+
56+
def updated_advisories(self):
57+
advisories = []
58+
for advisory_page in self.fetch_pages():
59+
advisories.extend(self.to_advisories(advisory_page))
60+
return self.batch_advisories(advisories)
61+
62+
def fetch_pages(self):
63+
tomcat_major_versions = {i[0] for i in self.version_api.get("org.apache.tomcat:tomcat")}
64+
for version in tomcat_major_versions:
65+
page_url = self.base_url.format(version)
66+
if create_etag(self, page_url, "ETag"):
67+
yield requests.get(page_url).content
68+
69+
def to_advisories(self, apache_tomcat_advisory_html):
70+
advisories = []
71+
page_soup = BeautifulSoup(apache_tomcat_advisory_html, features="lxml")
72+
pageh3s = page_soup.find_all("h3")
73+
vuln_headings = [i for i in pageh3s if "Fixed in Apache Tomcat" in i.text]
74+
for data in vuln_headings:
75+
fixed_version = data.text.split("Fixed in Apache Tomcat")[-1].strip()
76+
details_div = data.find_next_sibling()
77+
78+
for anchor_tag in details_div.find_all("a"):
79+
if "cve.mitre.org" not in anchor_tag["href"]:
80+
continue
81+
82+
cve_id = re.search(r"CVE-\d*-\d*", anchor_tag.text).group()
83+
references = []
84+
affected_packages = []
85+
paragraph = anchor_tag.find_parent()
86+
87+
while paragraph and "Affects:" not in paragraph.text:
88+
for ref in paragraph.find_all("a"):
89+
references.append(Reference(url=ref["href"]))
90+
91+
paragraph = paragraph.find_next_sibling()
92+
93+
if not paragraph:
94+
# At the end of details_div
95+
continue
96+
97+
for version_range in parse_version_ranges(paragraph.text):
98+
affected_packages.extend(
99+
[
100+
PackageURL(
101+
type="maven", namespace="apache", name="tomcat", version=version
102+
)
103+
for version in self.version_api.get("org.apache.tomcat:tomcat")
104+
if version in version_range
105+
]
106+
)
107+
108+
fixed_package = [
109+
PackageURL(
110+
type="maven", namespace="apache", name="tomcat", version=fixed_version
111+
)
112+
]
113+
114+
advisories.append(
115+
Advisory(
116+
summary="",
117+
impacted_package_urls=affected_packages,
118+
resolved_package_urls=fixed_package,
119+
cve_id=cve_id,
120+
vuln_references=references,
121+
)
122+
)
123+
124+
return advisories
125+
126+
127+
def parse_version_ranges(string):
128+
"""
129+
This method yields Rangespecifier objects obtained by
130+
parsing `string`.
131+
>> list(parse_version_ranges("Affects: 9.0.0.M1 to 9.0.0.M9"))
132+
[RangeSpecifier(<=9.0.0.M9,>=9.0.0.M1)]
133+
134+
>> list(parse_version_ranges("Affects: 9.0.0.M1"))
135+
[RangeSpecifier(>=9.0.0.M1<=9.0.0.M1)]
136+
137+
>> list(parse_version_ranges("Affects: 9.0.0.M1 to 9.0.0.M9, 1.2.3 to 3.4.5"))
138+
[RangeSpecifier(<=9.0.0.M9,>=9.0.0.M1), RangeSpecifier(<=3.4.5,>=1.2.3)]
139+
"""
140+
version_rng_txt = string.split("Affects:")[-1].strip()
141+
version_ranges = version_rng_txt.split(",")
142+
for version_range in version_ranges:
143+
if "to" in version_range:
144+
lower_bound, upper_bound = version_range.split("to")
145+
elif "-" in version_range and not any([i.isalpha() for i in version_range]):
146+
lower_bound, upper_bound = version_range.split("-")
147+
else:
148+
lower_bound = upper_bound = version_range
149+
150+
yield RangeSpecifier(">=" + lower_bound + "<=" + upper_bound)

‎vulnerabilities/tests/test_apache_tomcat.py‎

Lines changed: 227 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,227 @@
1+
# Copyright (c) nexB Inc. and others. All rights reserved.
2+
# http://nexb.com and https://github.com/nexB/vulnerablecode/
3+
# The VulnerableCode software is licensed under the Apache License version 2.0.
4+
# Data generated with VulnerableCode require an acknowledgment.
5+
#
6+
# You may not use this software except in compliance with the License.
7+
# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
8+
# Unless required by applicable law or agreed to in writing, software distributed
9+
# under the License is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES OR
10+
# CONDITIONS OF ANY KIND, either express or implied. See the License for the
11+
# specific language governing permissions and limitations under the License.
12+
#
13+
# When you publish or redistribute any data created with VulnerableCode or any VulnerableCode
14+
# derivative work, you must accompany this data with the following acknowledgment:
15+
#
16+
# Generated with VulnerableCode and provided on an 'AS IS' BASIS, WITHOUT WARRANTIES
17+
# OR CONDITIONS OF ANY KIND, either express or implied. No content created from
18+
# VulnerableCode should be considered or used as legal advice. Consult an Attorney
19+
# for any legal advice.
20+
# VulnerableCode is a free software from nexB Inc. and others.
21+
# Visit https://github.com/nexB/vulnerablecode/ for support and download.
22+
23+
import os
24+
from unittest.mock import MagicMock
25+
from unittest.mock import patch
26+
from unittest import TestCase
27+
28+
from packageurl import PackageURL
29+
30+
from vulnerabilities.data_source import Advisory
31+
from vulnerabilities.data_source import Reference
32+
from vulnerabilities.importers.apache_tomcat import ApacheTomcatDataSource
33+
34+
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
35+
TEST_DATA = os.path.join(BASE_DIR, "test_data", "apache_tomcat", "security-9.html")
36+
37+
38+
class TestApacheTomcatDataSource(TestCase):
39+
@classmethod
40+
def setUpClass(cls):
41+
data_source_cfg = {"etags": {}}
42+
mock_api = {"org.apache.tomcat:tomcat": ["9.0.0.M1", "9.0.0.M2", "8.0.0.M1", "6.0.0M2"]}
43+
with patch("vulnerabilities.importers.apache_tomcat.MavenVersionAPI"):
44+
with patch("vulnerabilities.importers.apache_tomcat.asyncio"):
45+
cls.data_src = ApacheTomcatDataSource(1, config=data_source_cfg)
46+
47+
cls.data_src.version_api = mock_api
48+
49+
def test_to_advisories(self):
50+
expected_advisories = sorted(
51+
[
52+
Advisory(
53+
summary="",
54+
impacted_package_urls=[
55+
PackageURL(
56+
type="maven",
57+
namespace="apache",
58+
name="tomcat",
59+
version="9.0.0.M1",
60+
qualifiers={},
61+
subpath=None,
62+
),
63+
PackageURL(
64+
type="maven",
65+
namespace="apache",
66+
name="tomcat",
67+
version="9.0.0.M2",
68+
qualifiers={},
69+
subpath=None,
70+
),
71+
],
72+
resolved_package_urls=[
73+
PackageURL(
74+
type="maven",
75+
namespace="apache",
76+
name="tomcat",
77+
version="9.0.0.M3",
78+
qualifiers={},
79+
subpath=None,
80+
)
81+
],
82+
vuln_references=[
83+
Reference(
84+
url="http://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2016-0763",
85+
reference_id="",
86+
),
87+
Reference(
88+
url="https://svn.apache.org/viewvc?view=rev&rev=1725926",
89+
reference_id="",
90+
),
91+
],
92+
cve_id="CVE-2016-0763",
93+
),
94+
Advisory(
95+
summary="",
96+
impacted_package_urls=[
97+
PackageURL(
98+
type="maven",
99+
namespace="apache",
100+
name="tomcat",
101+
version="8.0.0.M1",
102+
qualifiers={},
103+
subpath=None,
104+
)
105+
],
106+
resolved_package_urls=[
107+
PackageURL(
108+
type="maven",
109+
namespace="apache",
110+
name="tomcat",
111+
version="9.0.0.M3",
112+
qualifiers={},
113+
subpath=None,
114+
)
115+
],
116+
vuln_references=[
117+
Reference(
118+
url="http://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2015-5351",
119+
reference_id="",
120+
),
121+
Reference(
122+
url="https://svn.apache.org/viewvc?view=rev&rev=1720652",
123+
reference_id="",
124+
),
125+
Reference(
126+
url="https://svn.apache.org/viewvc?view=rev&rev=1720655",
127+
reference_id="",
128+
),
129+
],
130+
cve_id="CVE-2015-5351",
131+
),
132+
Advisory(
133+
summary="",
134+
impacted_package_urls=[
135+
PackageURL(
136+
type="maven",
137+
namespace="apache",
138+
name="tomcat",
139+
version="9.0.0.M1",
140+
qualifiers={},
141+
subpath=None,
142+
),
143+
PackageURL(
144+
type="maven",
145+
namespace="apache",
146+
name="tomcat",
147+
version="9.0.0.M2",
148+
qualifiers={},
149+
subpath=None,
150+
),
151+
],
152+
resolved_package_urls=[
153+
PackageURL(
154+
type="maven",
155+
namespace="apache",
156+
name="tomcat",
157+
version="9.0.0.M3",
158+
qualifiers={},
159+
subpath=None,
160+
)
161+
],
162+
vuln_references=[
163+
Reference(
164+
url="http://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2016-0706",
165+
reference_id="",
166+
),
167+
Reference(
168+
url="https://svn.apache.org/viewvc?view=rev&rev=1722799",
169+
reference_id="",
170+
),
171+
],
172+
cve_id="CVE-2016-0706",
173+
),
174+
Advisory(
175+
summary="",
176+
impacted_package_urls=[
177+
PackageURL(
178+
type="maven",
179+
namespace="apache",
180+
name="tomcat",
181+
version="9.0.0.M2",
182+
qualifiers={},
183+
subpath=None,
184+
)
185+
],
186+
resolved_package_urls=[
187+
PackageURL(
188+
type="maven",
189+
namespace="apache",
190+
name="tomcat",
191+
version="9.0.0.M3",
192+
qualifiers={},
193+
subpath=None,
194+
)
195+
],
196+
vuln_references=[
197+
Reference(
198+
url="http://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2016-0714",
199+
reference_id="",
200+
),
201+
Reference(
202+
url="https://svn.apache.org/viewvc?view=rev&rev=1725263",
203+
reference_id="",
204+
),
205+
Reference(
206+
url="https://svn.apache.org/viewvc?view=rev&rev=1725914",
207+
reference_id="",
208+
),
209+
],
210+
cve_id="CVE-2016-0714",
211+
),
212+
],
213+
key=lambda x: x.cve_id,
214+
)
215+
216+
with open(TEST_DATA) as f:
217+
found_advisories = self.data_src.to_advisories(f)
218+
219+
found_advisories.sort(key=lambda x: x.cve_id)
220+
221+
for i in range(len(found_advisories)):
222+
found_advisories[i].vuln_references.sort(key=lambda x: x.url)
223+
224+
for i in range(len(expected_advisories)):
225+
expected_advisories[i].vuln_references.sort(key=lambda x: x.url)
226+
227+
assert expected_advisories == found_advisories

0 commit comments

Comments
 (0)