Skip to content

Commit 8788014

Browse files
haikoscholhaikoschol
authored
Merge pull request #213 from sbs2001/ubuntu_usn_importer
Add ubuntu usn importer and it's tests
2 parents 3041144 + bfba423 commit 8788014

5 files changed

Lines changed: 381 additions & 0 deletions

File tree

‎vulnerabilities/importers/__init__.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,3 +35,4 @@
3535
from vulnerabilities.importers.redhat import RedhatDataSource
3636
from vulnerabilities.importers.gentoo import GentooDataSource
3737
from vulnerabilities.importers.openssl import OpenSSLDataSource
38+
from vulnerabilities.importers.ubuntu_usn import UbuntuUSNDataSource

‎vulnerabilities/importers/ubuntu_usn.py‎

Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,129 @@
1+
# Copyright (c) nexB Inc. and others. All rights reserved.
2+
# http://nexb.com and https://github.com/nexB/vulnerablecode/
3+
# The VulnerableCode software is licensed under the Apache License version 2.0.
4+
# Data generated with VulnerableCode require an acknowledgment.
5+
#
6+
# You may not use this software except in compliance with the License.
7+
# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
8+
# Unless required by applicable law or agreed to in writing, software distributed
9+
# under the License is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES OR
10+
# CONDITIONS OF ANY KIND, either express or implied. See the License for the
11+
# specific language governing permissions and limitations under the License.
12+
#
13+
# When you publish or redistribute any data created with VulnerableCode or any VulnerableCode
14+
# derivative work, you must accompany this data with the following acknowledgment:
15+
#
16+
# Generated with VulnerableCode and provided on an 'AS IS' BASIS, WITHOUT WARRANTIES
17+
# OR CONDITIONS OF ANY KIND, either express or implied. No content created from
18+
# VulnerableCode should be considered or used as legal advice. Consult an Attorney
19+
# for any legal advice.
20+
# VulnerableCode is a free software code scanning tool from nexB Inc. and others.
21+
# Visit https://github.com/nexB/vulnerablecode/ for support and download.
22+
23+
import bz2
24+
import dataclasses
25+
import json
26+
27+
import requests
28+
from packageurl import PackageURL
29+
30+
from vulnerabilities.data_source import DataSource
31+
from vulnerabilities.data_source import Advisory
32+
33+
34+
@dataclasses.dataclass
35+
class USNDBConfiguration:
36+
etags: list
37+
db_url: str
38+
39+
40+
class UbuntuUSNDataSource(DataSource):
41+
CONFIG_CLASS = USNDBConfiguration
42+
43+
def updated_advisories(self):
44+
advisories = []
45+
if self.create_etag(self.config.db_url):
46+
advisories.extend(self.to_advisories(fetch(self.config.db_url)))
47+
48+
return self.batch_advisories(advisories)
49+
50+
def create_etag(self, url):
51+
etag = requests.head(url).headers.get('etag')
52+
if not etag:
53+
return True
54+
55+
elif url in self.config.etags:
56+
if self.config.etags[url] == etag:
57+
return False
58+
59+
self.config.etags[url] = etag
60+
return True
61+
62+
@staticmethod
63+
def to_advisories(usn_db):
64+
advisories = []
65+
for usn in usn_db:
66+
usnid_data = get_usn_references(usn_db[usn]['id'])
67+
for release in usn_db[usn]['releases']:
68+
pkg_dict = usn_db[usn]['releases'][release]
69+
safe_purls = get_purls(pkg_dict)
70+
71+
for cve in usn_db[usn].get('cves', ['']):
72+
# The db sometimes contains entries like
73+
# {'cves': ['python-pgsql vulnerabilities', 'CVE-2006-2313', 'CVE-2006-2314']}
74+
# This `if` filters entries like 'python-pgsql vulnerabilities'
75+
if not cve.startswith('CVE-'):
76+
continue
77+
78+
advisories.append(Advisory(
79+
cve_id=cve,
80+
impacted_package_urls=[],
81+
resolved_package_urls=safe_purls,
82+
summary='',
83+
reference_urls=usnid_data['reference_url'],
84+
reference_ids=[usnid_data['reference_id']]))
85+
86+
return advisories
87+
88+
89+
def get_usn_references(usn_id):
90+
return {'reference_id': 'USN-' + usn_id,
91+
'reference_url': ['https://usn.ubuntu.com/{}/'.format(usn_id)]
92+
}
93+
94+
95+
def fetch(url):
96+
response = requests.get(url).content
97+
raw_data = bz2.decompress(response)
98+
99+
return json.loads(raw_data)
100+
101+
102+
def get_purls(pkg_dict):
103+
purls = set()
104+
for pkg_name in pkg_dict.get('sources', []):
105+
version = pkg_dict['sources'][pkg_name]['version']
106+
# The db sometimes contains entries like {'postgresql': {'version': ''}}
107+
# This `if` ignores such entries
108+
if not version:
109+
continue
110+
111+
purls.add(PackageURL(name=pkg_name,
112+
version=version,
113+
type='deb',
114+
namespace='ubuntu',
115+
))
116+
117+
for pkg_name in pkg_dict['binaries']:
118+
version = pkg_dict['binaries'][pkg_name]['version']
119+
# The db sometimes contains entries like {'postgresql': {'version': ''}}
120+
# This `if` ignores such entries
121+
if not version:
122+
continue
123+
124+
purls.add(PackageURL(name=pkg_name,
125+
version=version,
126+
type='deb',
127+
namespace='ubuntu',
128+
))
129+
return purls

‎vulnerabilities/migrations/0016_ubuntu_usn_importer.py‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
# Copyright (c) nexB Inc. and others. All rights reserved.
2+
# http://nexb.com and https://github.com/nexB/vulnerablecode/
3+
# The VulnerableCode software is licensed under the Apache License version 2.0.
4+
# Data generated with VulnerableCode require an acknowledgment.
5+
#
6+
# You may not use this software except in compliance with the License.
7+
# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
8+
# Unless required by applicable law or agreed to in writing, software distributed
9+
# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
10+
# CONDITIONS OF ANY KIND, either express or implied. See the License for the
11+
# specific language governing permissions and limitations under the License.
12+
#
13+
# When you publish or redistribute any data created with VulnerableCode or any VulnerableCode
14+
# derivative work, you must accompany this data with the following acknowledgment:
15+
#
16+
# Generated with VulnerableCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES
17+
# OR CONDITIONS OF ANY KIND, either express or implied. No content created from
18+
# VulnerableCode should be considered or used as legal advice. Consult an Attorney
19+
# for any legal advice.
20+
# VulnerableCode is a free software code scanning tool from nexB Inc. and others.
21+
# Visit https://github.com/nexB/vulnerablecode/ for support and download.
22+
23+
from django.db import migrations
24+
def add_ubuntu_usn_importer(apps, _) :
25+
26+
Importer = apps.get_model('vulnerabilities', 'Importer')
27+
Importer.objects.create(
28+
name='ubuntu_usn',
29+
license='',
30+
last_run=None,
31+
data_source='UbuntuUSNDataSource',
32+
data_source_cfg={'etags':{},'db_url':'https://usn.ubuntu.com/usn-db/database-all.json.bz2'}
33+
)
34+
35+
def remove_ubuntu_usn_importer(apps, _):
36+
Importer = apps.get_model('vulnerabilities', 'Importer')
37+
qs = Importer.objects.filter(name='ubuntu_usn')
38+
if qs:
39+
qs[0].delete()
40+
41+
class Migration(migrations.Migration):
42+
dependencies = [
43+
44+
('vulnerabilities', '0015_openssl_importer'),
45+
46+
]
47+
48+
operations = [
49+
50+
migrations.RunPython(add_ubuntu_usn_importer, remove_ubuntu_usn_importer),
51+
52+
]

‎vulnerabilities/tests/test_data/ubuntu_usn_db/database-all.json.bz2‎

4.05 KB
Binary file not shown.

‎vulnerabilities/tests/test_ubuntu_usn.py‎

Lines changed: 199 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,199 @@
1+
# Copyright (c) nexB Inc. and others. All rights reserved.
2+
# http://nexb.com and https://github.com/nexB/vulnerablecode/
3+
# The VulnerableCode software is licensed under the Apache License version 2.0.
4+
# Data generated with VulnerableCode require an acknowledgment.
5+
#
6+
# You may not use this software except in compliance with the License.
7+
# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
8+
# Unless required by applicable law or agreed to in writing, software distributed
9+
# under the License is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES OR
10+
# CONDITIONS OF ANY KIND, either express or implied. See the License for the
11+
# specific language governing permissions and limitations under the License.
12+
#
13+
# When you publish or redistribute any data created with VulnerableCode or any VulnerableCode
14+
# derivative work, you must accompany this data with the following acknowledgment:
15+
#
16+
# Generated with VulnerableCode and provided on an 'AS IS' BASIS, WITHOUT WARRANTIES
17+
# OR CONDITIONS OF ANY KIND, either express or implied. No content created from
18+
# VulnerableCode should be considered or used as legal advice. Consult an Attorney
19+
# for any legal advice.
20+
# VulnerableCode is a free software code scanning tool from nexB Inc. and others.
21+
# Visit https://github.com/nexB/vulnerablecode/ for support and download.
22+
23+
import bz2
24+
from collections import OrderedDict
25+
import json
26+
import os
27+
from unittest import TestCase
28+
from unittest.mock import MagicMock
29+
from unittest.mock import patch
30+
31+
from packageurl import PackageURL
32+
33+
from vulnerabilities.data_source import Advisory
34+
import vulnerabilities.importers.ubuntu_usn as ubuntu_usn
35+
36+
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
37+
TEST_DATA = os.path.join(BASE_DIR, 'test_data/', 'ubuntu_usn_db', 'database-all.json.bz2')
38+
39+
40+
class TestUbuntuUSNDataSource(TestCase):
41+
@classmethod
42+
def setUpClass(cls):
43+
data_src_cfg = {'etags': {}, 'db_url': 'http://exampledb.com'}
44+
cls.data_src = ubuntu_usn.UbuntuUSNDataSource(batch_size=1, config=data_src_cfg)
45+
with open(TEST_DATA, 'rb') as f:
46+
cls.raw_data = f.read()
47+
cls.db = json.loads(bz2.decompress(cls.raw_data))
48+
49+
def test_get_usn_references(self):
50+
51+
eg_usn = '435-1'
52+
expected_references = {
53+
'reference_id': 'USN-435-1',
54+
'reference_url': ['https://usn.ubuntu.com/435-1/'],
55+
}
56+
57+
found_references = ubuntu_usn.get_usn_references(eg_usn)
58+
assert found_references == expected_references
59+
60+
def test_fetch(self):
61+
62+
mock_response = MagicMock()
63+
mock_response.content = self.raw_data
64+
with patch('vulnerabilities.importers.ubuntu_usn.requests.get', return_value=mock_response):
65+
assert ubuntu_usn.fetch('www.db.com') == self.db
66+
67+
def test_get_purls(self):
68+
69+
eg_pkg_dict_1 = self.db['763-1']['releases']['hardy']
70+
eg_pkg_dict_2 = self.db['763-1']['releases']['dapper']
71+
eg_pkg_dict_3 = self.db['763-1']['releases']['intrepid']
72+
73+
exp_pkgs_1 = {
74+
PackageURL(
75+
type='deb',
76+
namespace='ubuntu',
77+
name='xine-lib',
78+
version='1.1.11.1-1ubuntu3.4',
79+
qualifiers=OrderedDict(),
80+
subpath=None,
81+
),
82+
PackageURL(
83+
type='deb',
84+
namespace='ubuntu',
85+
name='libxine1',
86+
version='1.1.11.1-1ubuntu3.4',
87+
qualifiers=OrderedDict(),
88+
subpath=None,
89+
),
90+
}
91+
exp_pkgs_2 = {
92+
PackageURL(
93+
type='deb',
94+
namespace='ubuntu',
95+
name='libxine-main1',
96+
version='1.1.1+ubuntu2-7.12',
97+
qualifiers=OrderedDict(),
98+
subpath=None,
99+
),
100+
PackageURL(
101+
type='deb',
102+
namespace='ubuntu',
103+
name='xine-lib',
104+
version='1.1.1+ubuntu2-7.12',
105+
qualifiers=OrderedDict(),
106+
subpath=None,
107+
),
108+
}
109+
exp_pkgs_3 = {
110+
PackageURL(
111+
type='deb',
112+
namespace='ubuntu',
113+
name='xine-lib',
114+
version='1.1.15-0ubuntu3.3',
115+
qualifiers=OrderedDict(),
116+
subpath=None,
117+
),
118+
PackageURL(
119+
type='deb',
120+
namespace='ubuntu',
121+
name='libxine1',
122+
version='1.1.15-0ubuntu3.3',
123+
qualifiers=OrderedDict(),
124+
subpath=None,
125+
),
126+
}
127+
128+
assert exp_pkgs_1 == ubuntu_usn.get_purls(eg_pkg_dict_1)
129+
assert exp_pkgs_2 == ubuntu_usn.get_purls(eg_pkg_dict_2)
130+
assert exp_pkgs_3 == ubuntu_usn.get_purls(eg_pkg_dict_3)
131+
132+
def test_to_advisories(self):
133+
134+
expected_advisories = {
135+
Advisory(
136+
summary='',
137+
impacted_package_urls=[],
138+
resolved_package_urls={
139+
PackageURL(
140+
type='deb',
141+
namespace='ubuntu',
142+
name='xine-lib',
143+
version='1.1.15-0ubuntu3.3',
144+
qualifiers=OrderedDict(),
145+
subpath=None,
146+
),
147+
PackageURL(
148+
type='deb',
149+
namespace='ubuntu',
150+
name='libxine1',
151+
version='1.1.15-0ubuntu3.3',
152+
qualifiers=OrderedDict(),
153+
subpath=None,
154+
),
155+
},
156+
reference_urls=['https://usn.ubuntu.com/763-1/'],
157+
reference_ids=['USN-763-1'],
158+
cve_id='CVE-2009-0698',
159+
),
160+
Advisory(
161+
summary='',
162+
impacted_package_urls=[],
163+
resolved_package_urls={
164+
PackageURL(
165+
type='deb',
166+
namespace='ubuntu',
167+
name='xine-lib',
168+
version='1.1.15-0ubuntu3.3',
169+
qualifiers=OrderedDict(),
170+
subpath=None,
171+
),
172+
PackageURL(
173+
type='deb',
174+
namespace='ubuntu',
175+
name='libxine1',
176+
version='1.1.15-0ubuntu3.3',
177+
qualifiers=OrderedDict(),
178+
subpath=None,
179+
),
180+
},
181+
reference_urls=['https://usn.ubuntu.com/763-1/'],
182+
reference_ids=['USN-763-1'],
183+
cve_id='CVE-2009-1274',
184+
),
185+
}
186+
found_advisories = set(self.data_src.to_advisories(self.db))
187+
188+
assert expected_advisories == found_advisories
189+
190+
def test_create_etag(self):
191+
assert self.data_src.config.etags == {}
192+
193+
mock_response = MagicMock()
194+
mock_response.headers = {'etag': '2131151243&2191'}
195+
196+
with patch('vulnerabilities.importers.ubuntu.requests.head', return_value=mock_response):
197+
assert self.data_src.create_etag('https://example.org')
198+
assert self.data_src.config.etags == {'https://example.org': '2131151243&2191'}
199+
assert not self.data_src.create_etag('https://example.org')

0 commit comments

Comments
 (0)